/*
 * Copyright (c) 2015-2021 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#ifndef _SKYWALK_OS_STATS_H_
#define _SKYWALK_OS_STATS_H_

#if defined(PRIVATE) || defined(BSD_KERNEL_PRIVATE)
#include <skywalk/os_channel.h>
#include <skywalk/os_nexus_private.h>
#include <skywalk/os_packet.h>
#include <netinet/in.h>
#include <netinet/in_stat.h>
#include <net/if.h>
#include <net/ethernet.h>

/*
 * [Intro]
 * This file defines new statistics infrastructure using enum and uin64_t array
 * for skywalk. BSD network stack statistics definition would be kept for
 * backward compatibility.
 *
 * This new scheme has several advantages over legacy statistics definition.
 *  1. Using array based data structure allows efficient folding with vector
 *     instructions.
 *  2. It also allows simple indexing of statistics field using enum value
 *     rather than struct field name.
 *  3. Coupled with descriptive string per statistics enum, it allows convenient
 *     printing.
 *
 * The new statistics enum and data structure follows naming convention as:
 *  1. It should resemble legacy names to make transition easy.
 *  2. It should be easily distinguishable from legacy data structure.
 *
 *
 * [Naming]
 * The new data structures are named with suffix _stats (plural) v.s legacy stat
 * and are always in snake case, e.g.
 *          New                      |   Legacy
 *          -------------------------+--------------------------
 *          ip_stats                 |   ipstat
 *          ip6_stats                |   ip6stat
 *          tcp_stats                |   tcpstat
 *          if_tcp_ecn_stats         |   if_tcp_ecn_stat
 *          if_tcp_ecn_perf_stats    |   if_tcp_ecn_perf_stat
 *          udp_stats                |   udpstat
 *
 *
 * Enums are named as <stats_family>_<type>, e.g.
 *          New                      |   Legacy
 *          -------------------------+--------------------------
 *          IP_STATS_TOTAL           |   ipstat.ips_total
 *          TCP_STATS_CONNATTEMPT    |   tcpstat.tcps_connattempt
 *
 *
 * [Usage]
 * Declare a statistics data structure:
 *
 *     struct ip_stats sample_ip_stats;
 *
 * To increment a statistics value by one:
 *
 *     STATS_INC(&sample_ip_stats, IP_STATS_TOTAL);
 *
 * To add to a statistics value by a certain value:
 *
 *     STATS_ADD(&sample_ip_stats, IP_STATS_TOTAL, n_pkts);
 *
 * To fold one statistics data structure into another one:
 *
 *     ip_stats_fold(&sample_ip_stats, &another_ip_stats);
 *
 * To reset statistics:
 *
 *     ip_stats_reset(&sample_ip_stats);
 *
 *
 * [Note]
 * Some legacy stats member has been removed for convenience reason, e.g.
 *
 *     ip6stat.ip6s_nxthist[256]
 *     ip6stat.ip6s_sources_sameif[SCOPE6_ID_MAX];
 *
 * are removed, as they don't have easy mapping into array based statistics
 * scheme and doesn't have explicit usage within the kernel.
 * That's been said, those are treated as exceptions and could be reconsidered
 * if they are needed in the future.
 *
 *
 * [Adding Stats]
 * To add new stats definition, a X-Macro table and STATS_REGISTER is needed.
 *
 *     #define NEW_STATS_TABLE(X)\
 *         X(sub_type_1, "sub_type_1_short_name", "sub_type_1_fmt_string")\
 *         X(sub_type_2, "sub_type_2_short_name", "sub_type_2_fmt_string")\
 *         X(__NEW_STATS_MAX, "", "")
 *
 *     STATS_REGISTER(new_stats, NEW_STATS);
 *
 *  where new_stats is the lower case name, and NEW_STATS is its upper case.
 *  The following would be generated by STATS_REGISTER Macro:
 *
 *     enum _new_stats {
 *         sub_type_1,
 *         sub_type_2,
 *         __NEW_STATS_MAX,
 *     };
 *
 *     struct new_stats {
 *         uint64_t _arr[__NEW_STATS_NEW];
 *     }
 *
 *     static inline const char* new_stats_str(enum _new_stats);
 *     static inline const char* new_stats_fmt(enum _new_stats);
 *
 *     static inline void __attribute__((always_inline))
 *         new_stats_fold(struct new_stats *dst, struct new_stats *src);
 *
 *     static inline void __attribute__((always_inline))
 *         new_stats_reset(struct new_stats *s);
 *
 * The fmt_string should conform to printf convention. Note here as uint64_t is
 * used to store stats value, `%llu` is used to format the print value (PRIu64
 * could be used but since long long is guaranteed to be 64bit long, casting
 * to llu should work well without introducing definitions from `inttypes.h`.
 */

/* BEGIN CSTYLED */

/* ip stats definitions */
#define IP_STATS_TABLE(X)       \
	/* Input stats */      \
	X(IP_STATS_TOTAL,		"TotalRcvd",	"\t%llu total packet received\n")       \
	X(IP_STATS_BADSUM,		"BadCsum",	"\t\t%llu bad header checksum\n")       \
	X(IP_STATS_RCV_SWCSUM,		"RcvSWCsumPkt",	"\t\t%llu header checksummed in software")      \
	X(IP_STATS_RCV_SWCSUM_BYTES,	"RcvSWCsumByte"," (%llu byte)\n")       \
	X(IP_STATS_TOOSMALL,		"DataTooSmall",	"\t\t%llu with size smaller than minimum\n")\
	X(IP_STATS_TOOSHORT,		"PktTooShort",	"\t\t%llu with data size < data length\n")      \
	X(IP_STATS_ADJ,			"TotalAdj",	"\t\t%llu with data size > data length\n")      \
	X(IP_STATS_ADJ_HWCSUM_CLR,	"HWCsumDisc",	"\t\t\t%llu packet forced to software checksum\n")      \
	X(IP_STATS_TOOLONG,		"TooLong",	"\t\t%llu with ip length > max ip packet size\n")       \
	X(IP_STATS_BADHLEN,		"BadHdrLen",	"\t\t%llu with header length < data size\n")    \
	X(IP_STATS_BADLEN,		"BadLen",	"\t\t%llu with data length < header length\n")  \
	X(IP_STATS_BADOPTIONS,		"BadOptions",	"\t\t%llu with bad options\n")  \
	X(IP_STATS_BADVERS,		"BadVer",	"\t\t%llu with incorrect version number\n")     \
	X(IP_STATS_FRAGMENTS,		"FragRcvd",	"\t\t%llu fragment received\n") \
	X(IP_STATS_FRAGDROPPED,		"FragDrop",	"\t\t\t%llu dropped (dup or out of space)\n")   \
	X(IP_STATS_FRAGTIMEOUT,		"FragTimeO",	"\t\t\t%llu dropped after timeout\n")   \
	X(IP_STATS_REASSEMBLED,		"Reassembled",	"\t\t\t%llu reassembled ok\n")  \
	X(IP_STATS_DELIVERED,		"Delivered",	"\t\t%llu packet for this host\n")      \
	X(IP_STATS_NOPROTO,		"UnkwnProto",	"\t\t%llu packet for unknown/unsupported protocol\n")   \
	X(IP_STATS_FORWARD,		"Fwd",		"\t\t%llu packet forwarded")    \
	X(IP_STATS_FASTFORWARD,		"FastFwd",	" (%llu packet fast forwarded)\n")      \
	X(IP_STATS_CANTFORWARD,		"CantFwd",	"\t\t%llu packet not forwardable\n")    \
	X(IP_STATS_NOTMEMBER,		"UnRegGroup",	"\t\t%llu packet received for unknown multicast group\n")       \
	X(IP_STATS_REDIRECTSENT,	"FwdSameNet",	"\t\t%llu redirect sent\n")     \
	X(IP_STATS_RXC_COLLISIONS,	"RxChnColl",	"\t\t%llu input packet not chained due to collision\n") \
	X(IP_STATS_RXC_CHAINED,		"RxChn",	"\t\t%llu input packet processed in a chain\n") \
	X(IP_STATS_RXC_NOTCHAIN,	"RxBypChn",	"\t\t%llu input packet unable to chain\n")      \
	X(IP_STATS_RXC_CHAINSZ_GT2,	"RxChnGT2",	"\t\t%llu input packet chain processed with length greater than 2\n")   \
	X(IP_STATS_RXC_CHAINSZ_GT4,	"RxChnGT4",	"\t\t%llu input packet chain processed with length greater than 4\n")   \
        \
	/* Output stats */      \
	X(IP_STATS_LOCALOUT,		"LocalOut",	"\t%llu packet sent from this host\n")  \
	X(IP_STATS_RAWOUT,		"RawPktOut",	"\t\t%llu packet sent with fabricated ip header\n")     \
	X(IP_STATS_ODROPPED,		"DropNoBuf",	"\t\t%llu output packet dropped due to no bufs, etc.\n")        \
	X(IP_STATS_NOROUTE,		"NoRoute",	"\t\t%llu output packet discarded due to no route\n")   \
	X(IP_STATS_FRAGMENTED,		"Fragmented",	"\t\t%llu output datagram fragmented\n")        \
	X(IP_STATS_OFRAGMENTS,		"OutFraged",	"\t\t%llu fragment created\n")  \
	X(IP_STATS_CANTFRAG,		"CantFrag",	"\t\t%llu datagram that can't be fragmented\n") \
	X(IP_STATS_NOGIF,		"NoGif",	"\t\t%llu tunneling packet that can't find gif\n")      \
	X(IP_STATS_BADADDR,		"BadAddr",	"\t\t%llu datagram with bad address in header\n")       \
	X(IP_STATS_PKTDROPCNTRL,	"DropNoCtl",	"\t\t%llu packet dropped due to no bufs for control data\n")    \
	X(IP_STATS_SND_SWCSUM,		"SndSWCsumPkt",	"\t\t%llu header checksummed in software")      \
	X(IP_STATS_SND_SWCSUM_BYTES,	"SndSWCsumByte"," (%llu byte)\n")       \
	X(IP_STATS_RXC_NOTLIST,		"IPInPkt",	"\t\t%llu input packet did not go through list processing path\n")      \
	X(__IP_STATS_MAX,		"",		"end of ip stats")

/* ipv6 stats definitions */
#define IP6_STATS_TABLE(X)      \
	/* Input Stats */       \
	X(IP6_STATS_TOTAL,		"TotalRcvd",	"\t%llu total packet received\n")       \
	X(IP6_STATS_TOOSMALL,		"DataTooSmall",	"\t\t%llu with size smaller than minimum\n")    \
	X(IP6_STATS_TOOSHORT,		"PktTooShort",	"\t\t%llu with data size < data length\n")      \
	X(IP6_STATS_ADJ,		"TotalAdj",	"\t\t%llu with data size > data length\n")      \
	X(IP6_STATS_ADJ_HWCSUM_CLR,	"HWCsumDisc",	"\t\t\t%llu packet forced to software checksum\n")      \
	X(IP6_STATS_BADOPTIONS,		"BadOptions",	"\t\t%llu with bad options\n")  \
	X(IP6_STATS_BADVERS,		"BadVer",	"\t\t%llu with incorrect version number\n")     \
	X(IP6_STATS_FRAGMENTS,		"FrafRcvd",	"\t\t%llu fragment received\n") \
	X(IP6_STATS_FRAGDROPPED,	"FragDrop",	"\t\t\t%llu dropped (dup or out of space)\n")   \
	X(IP6_STATS_FRAGTIMEOUT,	"FragTimeO",	"\t\t\t%llu dropped after timeout\n")   \
	X(IP6_STATS_FRAGOVERFLOW,	"FragOverFlow",	"\t\t\t%llu exceeded limit\n")  \
	X(IP6_STATS_REASSEMBLED,	"FragReassembled","\t\t\t%llu reassembled ok\n")                \
	X(IP6_STATS_ATMFRAG_RCVD,	"RAtomicFrag",	"\t\t\t%llu atomic fragments received\n")               \
	X(IP6_STATS_DELIVERED,		"Delivered",	"\t\t%llu packet for this host\n")      \
	X(IP6_STATS_FORWARD,		"Fwd",		"\t\t%llu packet forwarded\n")  \
	X(IP6_STATS_CANTFORWARD,	"CantFwd",	"\t\t%llu packet not forwardable\n")    \
	X(IP6_STATS_REDIRECTSENT,	"RedirectSent",	"\t\t%llu redirect sent\n")     \
	X(IP6_STATS_NOTMEMBER,		"NoMCGrp",	"\t\t%llu multicast packet which we don't join\n")      \
	X(IP6_STATS_EXTHDRTOOLONG,	"ExtHdrNotCont","\t\t\t%llu packet whose headers are not continuous\n") \
	X(IP6_STATS_NOGIF,		"NoGif",	"\t\t%llu tunneling packet that can't find gif\n")      \
	X(IP6_STATS_TOOMANYHDR,		"TooManyHdr",	"\t\t%llu packet discarded due to too may headers\n")   \
	X(IP6_STATS_FORWARD_CACHEHIT,	"FwdCacheHit",	"\t\t%llu forward cache hit\n") \
	X(IP6_STATS_FORWARD_CACHEMISS,	"FwdCacheMiss",	"\t\t%llu forward cache miss\n")        \
	X(IP6_STATS_PKTDROPCNTRL,	"DropNoCtl",	"\t\t%llu packet dropped due to no bufs for control data\n")    \
	/* Output stats */      \
	X(IP6_STATS_LOCALOUT,		"LocalOut",	"\t%llu packet sent from this host\n")  \
	X(IP6_STATS_RAWOUT,		"RawPktOut",	"\t\t%llu packet sent with fabricated ip header\n")     \
	X(IP6_STATS_ODROPPED,		"DropNoBuf",	"\t\t%llu output packet dropped due to no bufs, etc.\n")        \
	X(IP6_STATS_NOROUTE,		"NoRoute",	"\t\t%llu output packet discarded due to no route\n")   \
	X(IP6_STATS_FRAGMENTED,		"Fragmented",	"\t\t%llu output datagram fragmented\n")        \
	X(IP6_STATS_OFRAGMENTS,		"OutFraged",	"\t\t%llu fragment created\n")  \
	X(IP6_STATS_CANTFRAG,		"CantFrag",	"\t\t%llu datagram that can't be fragmented\n")\
	X(IP6_STATS_BADSCOPE,		"BadScope",	"\t\t%llu packet that violated scope rules\n")  \
	X(IP6_STATS_SOURCES_NONE,	"AddrSelFail",	"\t\t%llu failure of source address selection\n")       \
	X(IP6_STATS_DAD_COLLIDE,	"DADColl",	"\t\t%llu duplicate address detection collision\n")     \
	X(IP6_STATS_DAD_LOOPCOUNT,	"DADLoop",	"\t\t%llu duplicate address detection NS loop\n")       \
	X(IP6_STATS_SOURCES_SKIP6_EXPENSIVE_SECONDARY_IF,"Ign2ndIf",	"\t\t%llu time ignored source on secondary expensive I/F\n")    \
	X(__IP6_STATS_MAX,		"",		"end of ipv6 stats")

/* tcp stats definitions */
#define TCP_STATS_TABLE(X)      \
	/* Output stats */      \
	X(TCP_STATS_SNDTOTAL,		"SndTotalPkt",	"\t%llu packet sent\n") \
	X(TCP_STATS_SNDPACK,		"SndTotalDP",	"\t\t%llu data packet") \
	X(TCP_STATS_SNDBYTE,		"SndDataByte",	" (%llu byte)\n")       \
	X(TCP_STATS_SNDREXMITPACK,	"SndDPktReXmt",	"\t\t%llu data packet retransmitted")   \
	X(TCP_STATS_SNDREXMITBYTE,	"SndDByteReXmt"," (%llu byte)\n")       \
	X(TCP_STATS_MTURESENT,		"MTUReSnd",	"\t\t%llu resend initiated by MTU discovery\n") \
	X(TCP_STATS_SNDACKS,		"SndAck",	"\t\t%llu ack-only packet")     \
	X(TCP_STATS_DELACK,		"DelayAck",	" (%llu delayed)\n")    \
	X(TCP_STATS_SNDURG,		"SndURG",	"\t\t%llu URG only packet\n")   \
	X(TCP_STATS_SNDPROBE,		"SndWinProb",	"\t\t%llu window probe packet\n")       \
	X(TCP_STATS_SNDWINUP,		"SndWinUpd",	"\t\t%llu window update packet\n")      \
	X(TCP_STATS_SNDCTRL,		"SndCtlPkt",	"\t\t%llu control packet\n")    \
	X(TCP_STATS_FCHOLDPACKET,	"FlowCtlWh",	"\t\t%llu data packet sent after flow control\n")       \
	X(TCP_STATS_SYNCHALLENGE,	"SYNChallenge",	"\t\t%llu challenge ACK sent due to unexpected SYN\n")  \
	X(TCP_STATS_RSTCHALLENGE,	"RSTChallenge",	"\t\t%llu challenge ACK sent due to unexpected RST\n")  \
	X(TCP_STATS_SND_SWCSUM,		"SndSWCsumPkt",	"\t\t%llu checksummed in software")     \
	X(TCP_STATS_SND_SWCSUM_BYTES,	"SndSWCsumByte"," (%llu byte) over IPv4\n")     \
	X(TCP_STATS_SND6_SWCSUM,	"SndSWCsumPkt6","\t\t%llu checksummed in software")     \
	X(TCP_STATS_SND6_SWCSUM_BYTES,	"SndSWCsumByte6"," (%llu byte) over IPv6\n")    \
        \
	/* Input stats */       \
	X(TCP_STATS_RCVTOTAL,		"RcvTotalPkt",	"\t%llu packet received\n")     \
	X(TCP_STATS_RCVACKPACK,		"RcvAckPkt",	"\t\t%llu ack") \
	X(TCP_STATS_RCVACKBYTE,		"RcvAckByte",	" (for %llu byte)\n")   \
	X(TCP_STATS_RCVDUPACK,		"RcvDupAck",	"\t\t%llu duplicate ack\n")     \
	X(TCP_STATS_RCVACKTOOMUCH,	"RcvAckUnSnd",	"\t\t%llu ack for unsent data\n")       \
	X(TCP_STATS_RCVPACK,		"RcvPktInSeq",	"\t\t%llu packet received in-sequence") \
	X(TCP_STATS_RCVBYTE,		"RcvBInSeq",	" (%llu byte)\n")       \
	X(TCP_STATS_RCVDUPPACK,		"RcvDupPkt",	"\t\t%llu completely duplicate packet") \
	X(TCP_STATS_RCVDUPBYTE,		"RcvDupByte",	" (%llu byte)\n")       \
	X(TCP_STATS_PAWSDROP,		"PAWSDrop",	"\t\t%llu old duplicate packet\n")      \
	X(TCP_STATS_RCVMEMDROP,		"RcvMemDrop",	"\t\t%llu received packet dropped due to low memory\n") \
	X(TCP_STATS_RCVPARTDUPPACK,	"RcvDupData",	"\t\t%llu packet with some dup. data")  \
	X(TCP_STATS_RCVPARTDUPBYTE,	"RcvPDupByte",	" (%llu byte duped)\n") \
	X(TCP_STATS_RCVOOPACK,		"RcvOOPkt",	"\t\t%llu out-of-order packet") \
	X(TCP_STATS_RCVOOBYTE,		"RcvOOByte",	" (%llu byte)\n")               \
	X(TCP_STATS_RCVPACKAFTERWIN,	"RcvAftWinPkt",	"\t\t%llu packet of data after window") \
	X(TCP_STATS_RCVBYTEAFTERWIN,	"RcvAftWinByte"," (%llu byte)\n")               \
	X(TCP_STATS_RCVWINPROBE,	"RcvWinProbPkt","\t\t%llu window probe\n")              \
	X(TCP_STATS_RCVWINUPD,		"RcvWinUpdPkt",	"\t\t%llu window update packet\n")              \
	X(TCP_STATS_RCVAFTERCLOSE,	"RcvAftCloPkt",	"\t\t%llu packet received after close\n")       \
	X(TCP_STATS_BADRST,		"BadRST",	"\t\t%llu bad reset\n") \
	X(TCP_STATS_RCVBADSUM,		"RcvBadCsum",	"\t\t%llu discarded for bad checksum\n")        \
	X(TCP_STATS_RCV_SWCSUM,		"RcvSWCsumPkt",	"\t\t%llu checksummed in software")     \
	X(TCP_STATS_RCV_SWCSUM_BYTES,	"RcvSWCsumByte"," (%llu byte) over IPv4\n")     \
	X(TCP_STATS_RCV6_SWCSUM,	"RcvSWCsumPkt6","\t\t%llu checksummed in software")     \
	X(TCP_STATS_RCV6_SWCSUM_BYTES,	"RcvSWCsumByte6"," (%llu byte) over IPv6\n")    \
	X(TCP_STATS_RCVBADOFF,		"RcvBadOff",	"\t\t%llu discarded for bad header offset field\n")     \
	X(TCP_STATS_RCVSHORT,		"RcvTooShort",	"\t\t%llu discarded because packet too short\n")        \
	X(TCP_STATS_CONNATTEMPT,	"ConnInit",	"\t\t%llu discarded because packet too short\n")        \
        \
	/* Connection stats */  \
	X(TCP_STATS_ACCEPTS,		"ConnAcpt",	"\t%llu connection accept\n")   \
	X(TCP_STATS_BADSYN,		"BadSYN",	"\t%llu bad connection attempt\n")      \
	X(TCP_STATS_LISTENDROP,		"ListenDrop",	"\t%llu listen queue overflow\n")               \
	X(TCP_STATS_CONNECTS,		"ConnEst",	"\t%llu connection established (including accepts)\n")  \
	X(TCP_STATS_CLOSED,		"ConnClosed",	"\t%llu connection closed")     \
	X(TCP_STATS_DROPS,		"ConnDrop",	" (including %llu drop)\n")     \
	X(TCP_STATS_CACHEDRTT,		"RTTCacheUpd",	"\t\t%llu connection updated cached RTT on close\n")    \
	X(TCP_STATS_CACHEDRTTVAR,	"RTTVarCacheUpd","\t\t%llu connection updated cached RTT variance on close\n")  \
	X(TCP_STATS_CACHEDSSTHRESH,	"SSTholdCacheUpd","\t\t%llu connection updated cached ssthresh on close\n")     \
	X(TCP_STATS_CONNDROPS,		"EConnDrop",	"\t%llu embryonic connection dropped\n")        \
	X(TCP_STATS_RTTUPDATED,		"RTTUpdated",	"\t%llu segment updated rtt")   \
	X(TCP_STATS_SEGSTIMED,		"RTTTimed",	" (of %llu attempt)\n") \
	X(TCP_STATS_REXMTTIMEO,		"ReXmtTO",	"\t%llu retransmit timeout\n")  \
	X(TCP_STATS_TIMEOUTDROP,	"DropTO",	"\t\t%llu connection dropped by rexmit timeout\n")      \
	X(TCP_STATS_RXTFINDROP,		"ReXmtFINDrop",	"\t\t%llu connection dropped after retransmitting FIN\n")       \
	X(TCP_STATS_PERSISTTIMEO,	"PersistTO",	"\t%llu persist timeout\n")                     \
	X(TCP_STATS_PERSISTDROP,	"PersisStateTO","\t\t%llu connection dropped by persist timeout\n")     \
	X(TCP_STATS_KEEPTIMEO,		"KATO",		"\t%llu keepalive timeout\n")   \
	X(TCP_STATS_KEEPPROBE,		"KAProbe",	"\t\t%llu keepalive probe sent\n")      \
	X(TCP_STATS_KEEPDROPS,		"KADrop",	"\t\t%llu connection dropped by keepalive\n")   \
	X(TCP_STATS_PREDACK,		"PredAck",	"\t%llu correct ACK header prediction\n")       \
	X(TCP_STATS_PREDDAT,		"PredData",	"\t%llu correct data packet header prediction\n")       \
	X(TCP_STATS_PCBCACHEMISS,	"Pcb$Miss",	"\t%llu times pcb cahce miss")  \
        \
	/* SACK related stats */        \
	X(TCP_STATS_SACK_RECOVERY_EPISODE,	"SACKRecEpi",	"\t%llu SACK recovery episode\n")       \
	X(TCP_STATS_SACK_REXMITS,	        "SACKReXmt",	"\t%llu segment rexmit in SACK recovery episodes\n")    \
	X(TCP_STATS_SACK_REXMIT_BYTES,		"SACKReXmtB",	"\t%llu byte rexmit in SACK recovery episodes\n")       \
	X(TCP_STATS_SACK_RCV_BLOCKS,		"SACKRcvBlk",	"\t%llu SACK option (SACK blocks) received\n")  \
	X(TCP_STATS_SACK_SEND_BLOCKS,		"SACKSntBlk",	"\t%llu SACK option (SACK blocks) sent\n")      \
	X(TCP_STATS_SACK_SBOVERFLOW,		"SACKSndBlkOF",	"\t%llu SACK scoreboard overflow\n")    \
        \
	/* LRO related stats */ \
	X(TCP_STATS_COALESCED_PACK,		"CoalPkt",	"\t%llu LRO coalesced packet\n")        \
	X(TCP_STATS_FLOWTBL_FULL,		"FlowTblFull",	"\t\t%llu time LRO flow table was full\n")      \
	X(TCP_STATS_FLOWTBL_COLLISION,		"FlowTblColl",	"\t\t%llu collision in LRO flow table\n")               \
	X(TCP_STATS_LRO_TWOPACK,		"LRO2Pkt",	"\t\t%llu time LRO coalesced 2 packets\n")              \
	X(TCP_STATS_LRO_MULTPACK,		"LROMultiPkt",	"\t\t%llu time LRO coalesced 3 or 4 packets\n") \
	X(TCP_STATS_LRO_LARGEPACK,		"LROLargePkt",	"\t\t%llu time LRO coalesced 5 or more packets\n")      \
        \
	X(TCP_STATS_LIMITED_TXT,		"LimitedXmt",	"\t%llu limited transmit done\n")               \
	X(TCP_STATS_EARLY_REXMT,		"EarlyReXmt",	"\t%llu early retransmit done\n")               \
	X(TCP_STATS_SACK_ACKADV,		"SACKAdvAck",	"\t%llu time cumulative ack advanced along with SACK\n")        \
	X(TCP_STATS_PTO,			"ProbTO",	"\t%llu probe timeout\n")               \
	X(TCP_STATS_RTO_AFTER_PTO,		"RTOAfProb",	"\t\t%llu time retransmit timeout triggered after probe\n")     \
	X(TCP_STATS_PROBE_IF,			"ProbeIF",	"\t\t%llu time probe packets were sent for an interface\n")     \
	X(TCP_STATS_PROBE_IF_CONFLICT,		"ProbeIFConfl",	"\t\t%llu time couldn't send probe packets for an interface\n") \
	X(TCP_STATS_TLP_RECOVERY,		"TLPFastRecvr", "\t\t%llu time fast recovery after tail loss\n")        \
	X(TCP_STATS_TLP_RECOVERLASTPKT,		"TLPRecvrLPkt",	"\t\t%llu time recovered last packet \n")       \
	X(TCP_STATS_PTO_IN_RECOVERY,		"PTOInRecvr",	"\t\t%llu SACK based rescue retransmit\n")      \
        \
	/* ECN related stats */ \
	X(TCP_STATS_ECN_CLIENT_SETUP,		"ECNCliSetup",	"\t%llu client connection attempted to negotiate ECN\n")        \
	X(TCP_STATS_ECN_CLIENT_SUCCESS,		"ECNNegoSucc",	"\t\t%llu client connection successfully negotiated ECN\n")     \
	X(TCP_STATS_ECN_NOT_SUPPORTED,		"ECNSvrNoSupt", "\t\t%llu time graceful fallback to Non-ECN connection\n")      \
	X(TCP_STATS_ECN_LOST_SYN,		"ECNLOSSSYN",	"\t\t%llu time lost ECN negotiating SYN, followed by retransmission\n") \
	X(TCP_STATS_ECN_SERVER_SETUP,		"ECNSvrSetup",	"\t\t%llu server connection attempted to negotiate ECN\n")      \
	X(TCP_STATS_ECN_SERVER_SUCCESS,		"ECNSvrSucc",	"\t\t%llu server connection successfully negotiate ECN\n")      \
	X(TCP_STATS_ECN_LOST_SYNACK,		"ECNLossSYNACK","\t\t%llu time lost ECN negotiating SYN-ACK, followed by retransmission\n")     \
	X(TCP_STATS_ECN_RECV_CE,		"ECNRcv",	"\t\t%llu time received congestion experienced (CE) notification\n")    \
	X(TCP_STATS_ECN_RECV_ECE,		"ECNRcvECE",	"\t\t%llu time CWR was sent in response to ECE\n")      \
	X(TCP_STATS_ECN_SENT_ECE,		"ECNSndECE",	"\t\t%llu time sent ECE notification\n")        \
	X(TCP_STATS_ECN_CONN_RECV_CE,		"ECNConnRcvCE",	"\t\t%llu connection received CE atleast once\n")       \
	X(TCP_STATS_ECN_CONN_RECV_ECE,		"ECNConnRcvECE","\t\t%llu connection received ECE atleast once\n")      \
	X(TCP_STATS_ECN_CONN_PLNOCE,		"ECNConnPLNoCE","\t\t%llu connection using ECN have seen packet loss but no CE\n")      \
	X(TCP_STATS_ECN_CONN_PL_CE,		"ECNConnPLCE",	"\t\t%llu connection using ECN have seen packet loss and CE\n")         \
	X(TCP_STATS_ECN_CONN_NOPL_CE,		"ECNConnNoPLCE","\t\t%llu connection using ECN received CE but no packet loss\n")               \
	X(TCP_STATS_ECN_FALLBACK_SYNLOSS,	"ECNFbSYNLoss",	"\t\t%llu connection fell back to non-ECN due to SYN-loss\n")   \
	X(TCP_STATS_ECN_FALLBACK_REORDER,	"ECNFbReOrd",	"\t\t%llu connection fell back to non-ECN due to reordering\n") \
	X(TCP_STATS_ECN_FALLBACK_CE,		"ECNFbCE",	"\t\t%llu connection fell back to non-ECN due to excessive CE-markings\n")      \
	X(TCP_STATS_ECN_FALLBACK_DROPRST,	"ECNFbDrpRST",	"\t\t%llu ECN fallback caused by connection drop due to RST\n") \
	X(TCP_STATS_ECN_FALLBACK_DROPRXMT,	"ECNFbDrpReXmt","\t\t%llu ECN fallback due to drop after multiple retransmits\n")       \
	X(TCP_STATS_DETECT_REORDERING,		"ReOrdDetect",	"\t%llu time packet reordering was detected on a connection\n") \
	X(TCP_STATS_REORDERED_PKTS,		"ReOrdPkt",	"\t\t%llu time transmitted packets were reordered\n")   \
	X(TCP_STATS_DELAY_RECOVERY,		"DlyFastRecvr",	"\t\t%llu time fast recovery was delayed to handle reordering\n")               \
	X(TCP_STATS_AVOID_RXMT,			"AvoidReXmt",	"\t\t%llu time retransmission was avoided by delaying recovery\n")      \
	X(TCP_STATS_UNNECESSARY_RXMT,		"UnNeedReXmt",	"\t\t%llu retransmission not needed\n")\
        \
	/* DSACK related statistics */  \
	X(TCP_STATS_DSACK_SENT,			"DSACKSnd",	"\t%llu time DSACK option was sent\n")  \
	X(TCP_STATS_DSACK_RECVD,		"DSACKRcv",	"\t\t%llu time DSACK option was received\n")    \
	X(TCP_STATS_DSACK_DISABLE,		"DSACKDisable",	"\t\t%llu time DSACK was disabled on a connection\n")   \
	X(TCP_STATS_DSACK_BADREXMT,		"DSACKBadReXmt","\t\t%llu time recovered from bad retransmission using DSACK\n")        \
	X(TCP_STATS_DSACK_ACKLOSS,		"DSACKAckLoss",	"\t\t%llu time ignored DSACK due to ack loss\n")        \
	X(TCP_STATS_DSACK_RECVD_OLD,		"DSACKRcvOld",	"\t\t%llu time ignored old DSACK options\n")    \
	X(TCP_STATS_PMTUDBH_REVERTED,		"PMTUDBHRevert","\t%llu time PMTU Blackhole detection, size reverted\n")        \
	X(TCP_STATS_DROP_AFTER_SLEEP,		"DropAPSleep",	"\t%llu connection were dropped after long sleep\n")    \
        \
	/* TFO-related statistics */    \
	X(TCP_STATS_TFO_COOKIE_SENT,		"TFOCkSnd",	"\t%llu time a TFO-cookie has been announced\n")        \
	X(TCP_STATS_TFO_SYN_DATA_RCV,		"TFOSYNDataRcv","\t%llu SYN with data and a valid TFO-cookie have been received\n")     \
	X(TCP_STATS_TFO_COOKIE_REQ_RCV,		"TFOCkReqRcv",	"\t%llu SYN with TFO-cookie-request received\n")        \
	X(TCP_STATS_TFO_COOKIE_INVALID,		"TFOCkInv",	"\t%llu time an invalid TFO-cookie has been received\n")        \
	X(TCP_STATS_TFO_COOKIE_REQ,		"TFOCkReq",	"\t%llu time we requested a TFO-cookie\n")      \
	X(TCP_STATS_TFO_COOKIE_RCV,		"TFOCkRcv",	"\t\t%llu time the peer announced a TFO-cookie\n")      \
	X(TCP_STATS_TFO_SYN_DATA_SENT,		"TFOSYNDataSnd","\t%llu time we combined SYN with data and a TFO-cookie\n")     \
	X(TCP_STATS_TFO_SYN_DATA_ACKED,		"TDOSYNDataAck","\t\t%llu time our SYN with data has been acknowledged\n")      \
	X(TCP_STATS_TFO_SYN_LOSS,		"TFOSYNLoss",	"\t%llu time a connection-attempt with TFO fell back to regular TCP\n") \
	X(TCP_STATS_TFO_BLACKHOLE,		"TFOBlackhole",	"\t%llu time a TFO-connection blackhole'd\n")   \
	X(TCP_STATS_TFO_COOKIE_WRONG,		"TFOCkWrong",	"\t%llu time TFO-cookie we sent was wrong\n")   \
	X(TCP_STATS_TFO_NO_COOKIE_RCV,		"TFONoCkRcv",	"\t%llu time ee asked for a cookie but didn't get one\n")       \
	X(TCP_STATS_TFO_HEURISTICS_DISABLE,	"TFOHeuDisable","\t%llu time TFO got disabled due to heuristics\n")     \
	X(TCP_STATS_TFO_SNDBLACKHOLE,		"TFOSndBH",	"\t%llu time TFO got blackholed in the sending direction\n")    \
	X(TCP_STATS_MSS_TO_DEFAULT,		"MSSToDefault",	"\t%llu time maximum segment size was changed to default\n")    \
	X(TCP_STATS_MSS_TO_MEDIUM,		"MSSToMedium",	"\t%llu time maximum segment size was changed to medium\n")     \
	X(TCP_STATS_MSS_TO_LOW,			"MSSToLow",	"\t%llu time maximum segment size was changed to low\n")        \
        \
	/* TCP timer statistics */      \
	X(TCP_STATS_TIMER_DRIFT_LE_1_MS,	"TmrDriftLE1Ms",	"\t%llu timer drift less or equal to 1 ms\n")   \
	X(TCP_STATS_TIMER_DRIFT_LE_10_MS,	"TmrDriftLE10Ms",	"\t%llu timer drift less or equal to 10 ms\n")  \
	X(TCP_STATS_TIMER_DRIFT_LE_20_MS,	"TmrDriftLE20Ms",	"\t%llu timer drift less or equal to 20 ms\n")  \
	X(TCP_STATS_TIMER_DRIFT_LE_50_MS,	"TmrDriftLE50Ms",	"\t%llu timer drift less or equal to 50 ms\n")  \
	X(TCP_STATS_TIMER_DRIFT_LE_100_MS,	"TmrDriftLE100Ms",	"\t%llu timer drift less or equal to 100 ms\n") \
	X(TCP_STATS_TIMER_DRIFT_LE_200_MS,	"TmrDriftLE200Ms",	"\t%llu timer drift less or equal to 200 ms\n") \
	X(TCP_STATS_TIMER_DRIFT_LE_500_MS,	"TmrDriftLE500Ms",	"\t%llu timer drift less or equal to 500 ms\n") \
	X(TCP_STATS_TIMER_DRIFT_LE_1000_MS,	"TmrDriftLE1000Ms",	"\t%llu timer drift less or equal to 1000 ms\n")        \
	X(TCP_STATS_TIMER_DRIFT_GT_1000_MS,	"TmrDriftGT1000Ms",	"\t%llu timer drift greater than 1000 ms\n")    \
	X(TCP_STATS_USEDRTT,			"RTTUsed",	"\t%llu times RTT initialized from route\n")    \
	X(TCP_STATS_USEDRTTVAR,			"RTTVarUsed",	"\t%llu times RTTVAR initialized from rt\n")    \
	X(TCP_STATS_USEDSSTHRESH,		"SSTholdUsed",	"\t%llu times ssthresh initialized from rt\n")  \
	X(TCP_STATS_MINMSSDROPS,		"MinMssDrop",	"\t%llu average minmss too low drops\n")        \
	X(TCP_STATS_SNDREXMITBAD,		"BadReXmt",	"\t%llu unnecessary packet retransmissions\n")  \
        \
	/* SYN Cache relaetd stats */   \
	X(TCP_STATS_SC_ADDED,			"SCAdded",	"\t%llu entry added to syncache\n")     \
	X(TCP_STATS_SC_RETRANSMITTED,		"SCReXmt",	"\t%llu syncache entry was retransmitted\n")    \
	X(TCP_STATS_SC_DUPSYN,			"SCDupSYN",	"\t%llu duplicate SYN packet\n")        \
	X(TCP_STATS_SC_DROPPED,			"SCDrop",	"\t%llu could not reply to packet\n")   \
	X(TCP_STATS_SC_COMPLETED,		"SCCompl",	"\t%llu successful extraction of entry\n")      \
	X(TCP_STATS_SC_BUCKETOVERFLOW,		"SCBktOF",	"\t%llu syncache per-bucket limit hit\n")       \
	X(TCP_STATS_SC_CACHEOVERFLOW,		"SCOF",		"\t%llu syncache cache limit hit\n")    \
	X(TCP_STATS_SC_RESET,			"SCReset",	"\t%llu RST removed entry from syncache\n")     \
	X(TCP_STATS_SC_STALE,			"SCStale",	"\t%llu timed out or listen socket gone\n")     \
	X(TCP_STATS_SC_ABORTED,			"SCAbrt",	"\t%llu syncache entry aborted\n")      \
	X(TCP_STATS_SC_BADACK,			"SCBadAck",	"\t%llu removed due to bad ACK\n")      \
	X(TCP_STATS_SC_UNREACH,			"SCUnReach",	"\t%llu ICMP unreachable received\n")   \
	X(TCP_STATS_SC_ZONEFAIL,		"SCZoneFail",	"\t%llu zalloc() failed\n")     \
	X(TCP_STATS_SC_SENDCOOKIE,		"SCSndCookie",	"\t%llu SYN cookie sent\n")     \
	X(TCP_STATS_SC_RECVCOOKIE,		"SCRvcCookie",	"\t%llu SYN cookie received\n") \
        \
	/* Host Cache related stats */  \
	X(TCP_STATS_HC_ADDED,			"HCAdd",	"\t%llu entry added to hostcache\n")            \
	X(TCP_STATS_HC_BUCKETOVERFLOW,		"HCBktOF",	"\t%llu hostcache per bucket limit hit\n")      \
        \
	/* Misc. */     \
	X(TCP_STATS_BG_RCVTOTAL,		"RcvBkgrdPkt",	"\t%llu total background packets received\n")   \
	X(TCP_STATS_MSG_UNOPKTS,		"MsgUnOrdPkt",	"\t%llu unordered packet on TCP msg stream\n")  \
	X(TCP_STATS_MSG_UNOAPPENDFAIL,		"MsgUnOrdFail",	"\t%llu failed to append unordered pkt\n")      \
	X(TCP_STATS_MSG_SNDWAITHIPRI,		"MsgSndW8HPrio","\t%llu send waiting for high priority data\n") \
        \
	/* MPTCP Related stats */       \
	X(TCP_STATS_MP_SNDPACKS,		"MPSndPkt",	"\t%llu data packet sent\n")    \
	X(TCP_STATS_MP_SNDBYTES,		"MPSndByte",	"\t%llu data byte sent\n")      \
	X(TCP_STATS_MP_RCVTOTAL,		"MPRcvTotal",	"\t%llu data packet received\n")        \
	X(TCP_STATS_MP_RCVBYTES,		"MPRcvByte",	"\t%llu data byte received\n")  \
	X(TCP_STATS_INVALID_MPCAP,		"InvMPCap",	"\t%llu packet with an invalid MPCAP option\n") \
	X(TCP_STATS_INVALID_JOINS,		"InvMPJoin",	"\t%llu packet with an invalid MPJOIN option\n")        \
	X(TCP_STATS_MPCAP_FALLBACK,		"MPCapFail",	"\t%llu time primary subflow fell back to TCP\n")       \
	X(TCP_STATS_JOIN_FALLBACK,		"MPJoinFallBk",	"\t%llu time secondary subflow fell back to TCP\n")     \
	X(TCP_STATS_ESTAB_FALLBACK,		"EstFallBk",	"\t%llu DSS option drop\n")     \
	X(TCP_STATS_INVALID_OPT,		"InvOpt",	"\t%llu other invalid MPTCP option\n")  \
	X(TCP_STATS_MP_REDUCEDWIN,		"MPReducedWin",	"\t%llu time the MPTCP subflow window was reduced\n")   \
	X(TCP_STATS_MP_BADCSUM,			"MPBadCSum",	"\t%llu bad DSS checksum\n")    \
	X(TCP_STATS_MP_OODATA,			"MPOOData",	"\t%llu time received out of order data\n")     \
	X(TCP_STATS_MP_OUTOFWIN,		"MPOutOfWin",	"\t%llu Packet lies outside the shared recv window\n")  \
	X(TCP_STATS_JOIN_RXMTS,			"JoinAckReXmt",	"\t%llu join ack retransmits\n")        \
	X(TCP_STATS_TAILLOSS_RTO,		"TailLossTRO",	"\t%llu RTO due to tail loss\n")        \
	X(TCP_STATS_RECOVERED_PKTS,		"RecoveryPkt",	"\t%llu recovered after loss\n")        \
	X(TCP_STATS_NOSTRETCHACK,		"NoStrechAck",	"\t%llu disabled stretch ack algorithm on a connection\n")      \
	X(TCP_STATS_RESCUE_RXMT,		"SACKRsqReXmt", "\t%llu SACK rescue retransmit\n")      \
        \
	/* MPTCP Subflow selection stats */     \
	X(TCP_STATS_MP_SWITCHES,		"MPSwitches",	"\t%llu subflow switch\n")      \
	X(TCP_STATS_MP_SEL_SYMTOMSD,		"MPSelSymp",	"\t%llu subflow switch due to advisory\n")      \
	X(TCP_STATS_MP_SEL_RTT,			"MPSelRTT",	"\t%llu subflow switch due to rtt\n")   \
	X(TCP_STATS_MP_SEL_RTO,			"MPSelRTO",	"\t%llu subflow switch due to rto\n")   \
	X(TCP_STATS_MP_SEL_PEER,		"MPSelPeer",	"\t%llu subflow switch due to peer\n")  \
	X(TCP_STATS_MP_NUM_PROBES,		"MPProbe",	"\t%llu number of subflow probe\n")     \
	X(TCP_STATS_MP_VERDOWNGRADE,		"MPVerDowngrd",	"\t%llu times MPTCP version downgrade\n")       \
        \
	/* Miscellaneous statistics */  \
	X(TCP_STATS_TW_PCBCOUNT,		"TWPcbCount",	"\t%llu pcbs in time-wait state\n")     \
        \
	X(__TCP_STATS_MAX,			"",		"end of tcp stats")

#define UDP_STATS_TABLE(X)                                              \
	/* Input stats */       \
	X(UDP_STATS_IPACKETS,			"RcvPkt",		"\t%llu datagram received\n")   \
	X(UDP_STATS_HDROPS,			"HdrDrop",		"\t\t%llu with incomplete header\n")    \
	X(UDP_STATS_BADSUM,			"BadCsum",		"\t\t%llu with bad data length field\n")        \
	X(UDP_STATS_BADLEN,			"BadLen",		"\t\t%llu with bad checksum\n") \
	X(UDP_STATS_NOSUM,			"NoCsum",		"\t\t%llu with no checksum\n")  \
	X(UDP_STATS_RCV_SWCSUM,			"RcvSWCsum",		"\t\t%llu checksummed in software")     \
	X(UDP_STATS_RCV_SWCSUM_BYTES,		"RcvSWCsumBytes",	" (%llu bytes) over IPv4\n")    \
	X(UDP_STATS_RCV6_SWCSUM,		"RcvSWCsum",		"\t\t%llu checksummed in software")     \
	X(UDP_STATS_RCV6_SWCSUM_BYTES,		"RcvSWCsumBytes",	" (%llu bytes) over IPv6\n")    \
	X(UDP_STATS_NOPORT,			"NoPort",		"\t\t%llu dropped due to no socket\n")  \
	X(UDP_STATS_NOPORTBCAST,		"NoPortBCast",		"\t\t%llu broadcast/multicast datagram undelivered\n")  \
	X(UDP_STATS_FILTERMCAST,		"FilterMCast",		"\t\t%llu time multicast source filter matched\n")      \
	X(UDP_STATS_FULLSOCK,			"FullSock",		"\t\t%llu dropped due to full socket buffers\n")        \
	X(UDP_STATS_PCBCACHEMISS,		"PCBCacheMiss",		"\t\t%llu not for hashed pcb\n")        \
	X(UDP_STATS_PCBHASHMISS,		"PCBHashMiss",		"\t\t%llu input packets not for hashed pcb")    \
        \
	/* Output stats */      \
	X(UDP_STATS_OPACKETS,			"SndPkt",		"\t%llu datagram output\n")     \
	X(UDP_STATS_SND_SWCSUM,			"SndSWCsum",		"\t\t%llu checksummed in software")     \
	X(UDP_STATS_SND_SWCSUM_BYTES,		"SndSWCsumBytes",	" (%llu bytes) over IPv4\n")    \
	X(UDP_STATS_SND6_SWCSUM,		"SndSWCsum6",		"\t\t%llu checksummed in software")     \
	X(UDP_STATS_SND6_SWCSUM_BYTES,		"SndSWCsumBytes6",	" (%llu bytes) over IPv6\n")    \
	X(UDP_STATS_FASTOUT,			"SndFastPath",		"\t\t%llu output packets on fast path\n")       \
	X(UDP_STATS_NOPORTMCAST,		"SndNoPortMCast",	"\t\t%llu output no socket on port, multicast\n")       \
        \
	X(__UDP_STATS_MAX,			"",			"end of UDP stats")

#define QUIC_STATS_TABLE(X)                                      \
	/* Tx */        \
	X(QUIC_STATS_SNDPKT,			"SndTotalPkt",		"\t%llu packets sent")  \
	X(QUIC_STATS_SNDBYTE,			"SndTotalByte",		" (%llu bytes)\n")      \
	/* Tx Stream */ \
	X(QUIC_STATS_SNDSTREAMFRAME,		"SndStreamFrame",	"\t\tSTREAM\n\t\t\t%llu stream frames sent")    \
	X(QUIC_STATS_SNDSTREAMBYTE,		"SndStreamByte",	" (%llu bytes)\n")      \
	X(QUIC_STATS_SNDSTREAMRESET,		"SndStreamReset",	"\t\t\t%llu RESET_STREAM frames sent\n")        \
	X(QUIC_STATS_SNDSTOPSENDING,		"SndStopSending",	"\t\t\t%llu STOP_SENDING frames sent\n")        \
	X(QUIC_STATS_SNDSTREAMBLKFRAME,		"SndStreamBlockedFrame","\t\t\t%llu STREAM_BLOCKED frames sent\n")      \
	X(QUIC_STATS_SNDSTMDATABLKFRAME,	"SndStreamDataBlocked",	"\t\t\t%llu STREAM_DATA_BLOCKED frames sent\n") \
	/* Tx CRYPTO */ \
	X(QUIC_STATS_SNDINITCRYPTOFRAME,	"SndInitCryptoFrame",	"\t\tCRYPTO\n\t\t\t%llu initial CRYPTO frames sent")    \
	X(QUIC_STATS_SNDINITCRYPTOBYTE,		"SndInitCryptoByte",	" (%llu bytes)\n")      \
	X(QUIC_STATS_SNDHDSHKCRYPTOFRAME,	"SndHdShkCryptoFrame",	"\t\t\t%llu handshake CRYPTO frames sent")      \
	X(QUIC_STATS_SNDHDSHKCRYPTOBYTE,	"SndHdShkCryptoByte",	" (%llu bytes)\n")      \
	X(QUIC_STATS_SND1RTTCRYPTOFRAME,	"Snd1RttCryptoFrame",	"\t\t\t%llu 1-RTT CRYPTO frames sent")  \
	X(QUIC_STATS_SND1RTTCRYPTOBYTE,		"Snd1RttCryptoByte",	" (%llu bytes)\n")      \
	X(QUIC_STATS_SND0RTTCRYPTOFRAME,	"Snd1RttCryptoFrame",	"\t\t\t%llu 0-RTT CRYPTO frames sent")  \
	X(QUIC_STATS_SND0RTTCRYPTOBYTE,		"Snd1RttCryptoByte",	" (%llu bytes)\n")      \
	X(QUIC_STATS_SNDCRYPTOREXMTFRAME,	"SndReXmtCryptoFrame",	"\t\t\t%llu CRYPTO frames retransmitted")       \
	X(QUIC_STATS_SNDCRYPTOREXMTBYTE,	"SndReXmtCryptoByte",	" (%llu bytes)\n")      \
	X(QUIC_STATS_SNDDATABLKFRAME,		"SndDataBlockedFrame",	"\t\t%llu DATA_BLOCKED frames sent\n")  \
	X(QUIC_STATS_SNDREXMTPKT,		"SndReXmtPkt",		"\t\t%llu packets retransmitted")       \
	X(QUIC_STATS_SNDREXMTBYTE,		"SndReXmtByte",		" (%llu bytes)\n")      \
	X(QUIC_STATS_SNDLOSTPKT,		"SndLostPkt",		"\t\t%llu packets lost")        \
	X(QUIC_STATS_SNDLOSTBYTE,		"SndLostByte",		" (%llu bytes)\n")      \
	/* End of Tx */ \
	/* Rx */        \
	X(QUIC_STATS_RCVPKT,			"RcvTotalPkt",		"\t%llu packets received")      \
	X(QUIC_STATS_RECVBYTE,			"RcvTotalByte",		" (%llu bytes)\n")      \
	/* Rx Stream */ \
	X(QUIC_STATS_RCVSTREAMFRAME,		"RcvStreamFrame",	"\t\tSTREAM\n\t\t\t%llu stream frames received")        \
	X(QUIC_STATS_RCVSTREAMBYTE,		"RcvStreamByte",	" (%llu bytes)\n")      \
	X(QUIC_STATS_RCVSTREAMRESET,		"RcvStreamReset",	"\t\t\t%llu RESET_STREAM frames received\n")    \
	X(QUIC_STATS_RCVSTOPSENDING,		"RcvStopSending",	"\t\t\t%llu STOP_SENDING frames received\n")    \
	X(QUIC_STATS_RCVSTREAMBLKFRAME,		"RcvStreamBlockedFrame","\t\t\t%llu STREAM_BLOCKED frames received\n")  \
	X(QUIC_STATS_RCVSTMDATABLKFRAME,	"RcvStreamDataBlocked",	"\t\t\t%llu STREAM_DATA_BLOCKED frames received\n")     \
	/* Rx CRYPTO */ \
	X(QUIC_STATS_RCVINITCRYPTOFRAME,	"RcvInitCryptoFrame",	"\t\tCRYPTO\n\t\t\t%llu initial CRYPTO frames received")        \
	X(QUIC_STATS_RCVINITCRYPTOBYTE,		"RcvInitCryptoByte",	" (%llu bytes)\n")      \
	X(QUIC_STATS_RCVHDSHKCRYPTOFRAME,	"RcvHdShkCryptoFrame",	"\t\t\t%llu handshake CRYPTO frames received")  \
	X(QUIC_STATS_RCVHDSHKCRYPTOBYTE,	"RcvHdShkCryptoByte",	" (%llu bytes)\n")      \
	X(QUIC_STATS_RCV1RTTCRYPTOFRAME,	"Rcv1RttCryptoFrame",	"\t\t\t%llu 1-RTT CRYPTO frames received")      \
	X(QUIC_STATS_RCV1RTTCRYPTOBYTE,		"Rcv1RttCryptoByte",	" (%llu bytes)\n")      \
	X(QUIC_STATS_RCV0RTTCRYPTOFRAME,	"Rcv0RttCryptoFrame",	"\t\t\t%llu 0-RTT CRYPTO frames received")      \
	X(QUIC_STATS_RCV0RTTCRYPTOBYTE,		"Rcv0RttCryptoByte",	" (%llu bytes)\n")      \
	X(QUIC_STATS_RCVDATABLKFRAME,		"RcvDataBlockedFrame",	"\t\t%llu DATA_BLOCKED frames received\n")      \
	X(QUIC_STATS_RCVREORDERPKT,		"RcvReorderedPkt",	"\t\t%llu packets received reordered")  \
	X(QUIC_STATS_RCVREORDERBYTE,		"RcvReorderedByte",	" (%llu bytes)\n")      \
	/* Connections */       \
	X(QUIC_STATS_CONNECTS,			"ConnEst",		"\t%llu connections established\n")     \
	X(QUIC_STATS_SNDCCR,			"SndCCR",		"\t\t%llu connection close reasons sent\n")     \
	X(QUIC_STATS_SNDCCRINTERROR,		"SndCCRInternalError",	"\t\t\t%llu INTERNAL_ERROR sent\n")     \
	X(QUIC_STATS_SNDCCRSVRBUSY,		"SndCCRSererBusy",	"\t\t\t%llu SERVER_BUSY sent\n")        \
	X(QUIC_STATS_SNDCCRFLOWCTLERR,		"SndCCRFlowCtl",	"\t\t\t%llu FLOW_CONTROL_ERROR sent\n") \
	X(QUIC_STATS_SNDCCRSTREAMLIMIT,		"SndCCRStreamLimit",	"\t\t\t%llu STREAM_LIMIT_ERROR sent\n") \
	X(QUIC_STATS_SNDCCRSTREAMSTATE,		"SndCCRStreamState",	"\t\t\t%llu STREAM_STATE_ERROR sent\n") \
	X(QUIC_STATS_SNDCCRFINALOFFSET,		"SndCCRFinalOffset",	"\t\t\t%llu FINAL_SIZE_ERROR sent\n")   \
	X(QUIC_STATS_SNDCCRFRAMEENCODING,	"SndCCRFrameEncoding",	"\t\t\t%llu FRAME_ENCODING_ERROR sent\n")       \
	X(QUIC_STATS_SNDCCRTRNASPARAMS,		"SndCCRTransportParams","\t\t\t%llu TRANSPORT_PARAMETER_ERROR sent\n")  \
	X(QUIC_STATS_SNDCCRVERSIONNEGO,		"SndCCRVersionNego",	"\t\t\t%llu VERSION_NEGOTIATION_ERROR sent\n")  \
	X(QUIC_STATS_SNDCCRPROTOVIOLATION,	"SndCCRProtoViolation",	"\t\t\t%llu PROTOCOL_VIOLATION sent\n") \
	X(QUIC_STATS_SNDCCRINVALMIGRATION,	"SndCCRInvalMigration",	"\t\t\t%llu INVALID_MIGRATION sent\n")  \
	X(QUIC_STATS_SNDCCRCRYPTO,		"SndCCRCryptoError",	"\t\t\t%llu CRYPTO_ERROR sent\n")       \
	X(QUIC_STATS_RCVCCR,			"RcvCCR",		"\t\t%llu connection close reasons received\n") \
	X(QUIC_STATS_RCVCCRINTERROR,		"RcvCCRInternalError",	"\t\t\t%llu INTERNAL_ERROR received\n") \
	X(QUIC_STATS_RCVCCRSVRBUSY,		"RcvCCRSererBusy",	"\t\t\t%llu SERVER_BUSY received\n")    \
	X(QUIC_STATS_RCVCCRFLOWCTLERR,		"RcvCCRFlowCtl",	"\t\t\t%llu FLOW_CONTROL_ERROR received\n")     \
	X(QUIC_STATS_RCVCCRSTREAMLIMIT,		"RcvCCRStreamLimit",	"\t\t\t%llu STREAM_LIMIT_ERROR received\n")     \
	X(QUIC_STATS_RCVCCRSTREAMSTATE,		"RcvCCRStreamState",	"\t\t\t%llu STREAM_STATE_ERROR received\n")     \
	X(QUIC_STATS_RCVCCRFINALOFFSET,		"RcvCCRFinalOffset",	"\t\t\t%llu FINAL_SIZE_ERROR received\n")       \
	X(QUIC_STATS_RCVCCRFRAMEENCODING,	"RcvCCRFrameEncoding",	"\t\t\t%llu FRAME_ENCODING_ERROR received\n")   \
	X(QUIC_STATS_RCVCCRTRNASPARAMS,		"RcvCCRTransportParams","\t\t\t%llu TRANSPORT_PARAMETER_ERROR received\n")      \
	X(QUIC_STATS_RCVCCRVERSIONNEGO,		"RcvCCRVersionNego",	"\t\t\t%llu VERSION_NEGOTIATION_ERROR received\n")      \
	X(QUIC_STATS_RCVCCRPROTOVIOLATION,	"RcvCCRProtoViolation",	"\t\t\t%llu PROTOCOL_VIOLATION received\n")     \
	X(QUIC_STATS_RCVCCRINVALMIGRATION,	"RcvCCRInvalMigration",	"\t\t\t%llu INVALID_MIGRATION received\n")      \
	X(QUIC_STATS_RCVCCRCRYPTO,		"RcvCCRCryptoError",	"\t\t\t%llu CRYPTO_ERROR received\n")   \
	X(QUIC_STATS_SNDECT0,			"SndECT0",		"\t%llu ECT0 sent\n")   \
	X(QUIC_STATS_RCVECT0,			"RcvECT0",		"\t%llu ECT0 received\n")       \
	X(QUIC_STATS_SNDECT1,			"SndECT1",		"\t%llu ECT1 sent\n")   \
	X(QUIC_STATS_RCVECT1,			"RcvECT1",		"\t%llu ECT1 received\n")       \
	X(QUIC_STATS_SNDECTCE,			"SndECTCE",		"\t%llu ECT-CE sent\n") \
	X(QUIC_STATS_RCVECTCE,			"RcvECTCE",		"\t%llu ECT-CE received\n")     \
	X(QUIC_STATS_REXMTTIMEOUT,		"ReXmtTimeOut",		"\t%llu retransmit timeout\n")  \
	X(QUIC_STATS_KEEPALIVETTIMEOUT,		"KeepAliveTimeOut",	"\t%llu keepalive timeout\n")   \
	X(QUIC_STATS_PTO,			"ProbeTimeOut",		"\t%llu probe timeout\n")       \
        \
	X(__QUIC_STATS_MAX,			"",			"end of quic stats\n")

#define NETIF_STATS_TABLE(X)                                            \
	/* Rx stats */  \
	X(NETIF_STATS_RX_PACKETS,		"RxPackets",		"\t%llu total Rx packets\n")    \
	X(NETIF_STATS_RX_IRQ,			"RxIRQ",		"\t\t%llu interrupts\n")        \
	X(NETIF_STATS_RX_IRQ_MIT,		"RxIRQMIT",		"\t\t\t%llu interrupt mitigation thread wakeup\n")      \
	X(NETIF_STATS_RX_IRQ_BUSY,		"RxIRQBusy",		"\t\t\t%llu interrupt notify return busy\n")    \
	X(NETIF_STATS_RX_IRQ_AGAIN,		"RxIRQAgain",		"\t\t\t%llu interrupt notify return retry again\n")     \
	X(NETIF_STATS_RX_IRQ_ERR,		"RxIRQErr",		"\t\t\t%llu interrupt notify return error\n")   \
	X(NETIF_STATS_RX_COPY_SUM,		"RxCopySum",		"\t\t%llu copy+checksumed\n")   \
	X(NETIF_STATS_RX_COPY_DIRECT,		"RxCopyDirect",         "\t\t%llu copy from pkt\n")     \
	X(NETIF_STATS_RX_COPY_MBUF,		"RxCopyMbuf",		"\t\t%llu copy from mbuf\n")    \
	X(NETIF_STATS_RX_COPY_ATTACH,		"RxCopyAttach",         "\t\t%llu copy by attaching mbuf under pkt\n")  \
	X(NETIF_STATS_RX_SYNC,			"RxSYNC",		"\t\t%llu sync\n")      \
        \
	/* Tx stats */  \
	X(NETIF_STATS_TX_PACKETS,		"TxPkt",		"\t%llu total Tx packets\n")    \
	X(NETIF_STATS_TX_IRQ,			"TxIRQ",		"\t\t%llu interupts\n") \
	X(NETIF_STATS_TX_IRQ_MIT,		"TxIRQMIT",		"\t\t\t%llu interrupt mitigation thread wakeup\n")      \
	X(NETIF_STATS_TX_IRQ_BUSY,		"TxIRQBusy",		"\t\t\t%llu interrupt notify return busy\n")    \
	X(NETIF_STATS_TX_IRQ_AGAIN,		"TxIRQAgain",		"\t\t\t%llu interrupt notify return retry again\n")     \
	X(NETIF_STATS_TX_IRQ_ERR,		"TxIRQErr",		"\t\t\t%llu interrupt notify return error\n")   \
	X(NETIF_STATS_TX_COPY_SUM,		"TxCopySum",		"\t\t%llu copy+checksumed\n")   \
	X(NETIF_STATS_TX_COPY_DIRECT,		"TxCopyDirect",		"\t\t%llu copy from pkt\n")     \
	X(NETIF_STATS_TX_COPY_MBUF,		"TxCopyMbuf",		"\t\t%llu copy from mbuf\n")    \
	X(NETIF_STATS_TX_SYNC,			"TxSYNC",		"\t\t%llu sync\n")      \
	X(NETIF_STATS_TX_REPL,			"TxRepl",		"\t\t%llu pool replenished\n")  \
	X(NETIF_STATS_TX_DROP_ENQ_AQM,		"TxDropEnqueueAQM",     "\t\t%llu dropped due to AQM enqueue failure\n")        \
	X(NETIF_STATS_GSO_SEG,			"GSOSegments",          "\t\t%llu GSO segments created\n") \
	X(NETIF_STATS_GSO_PKT,			"GSOPackets",           "\t\t%llu GSO packets \n") \
	X(NETIF_STATS_GSO_PKT_DROP_NOMEM,	"GSODropNoMem",         "\t\t%llu GSO packet dropped due to allocation failure\n") \
	X(NETIF_STATS_GSO_PKT_DROP_NA_INACTIVE,	"GSODropNaInactive",    "\t\t%llu GSO packet dropped due to inactive netif\n") \
	X(NETIF_STATS_GSO_PKT_DROP_BADLEN,	"GSODropBadLen",        "\t\t%llu GSO packet dropped due to bad packet length\n") \
	X(NETIF_STATS_GSO_PKT_DROP_NONTCP,	"GSODropNonTcp",        "\t\t%llu GSO packet dropped as it is not a TCP packet\n") \
        \
	X(NETIF_STATS_DROP,			"Drop",			"\t%llu dropped\n")     \
	X(NETIF_STATS_DROP_NOMEM_BUF,		"DropNoMemBuf",		"\t\t%llu dropped due to packet alloc failure\n")       \
	X(NETIF_STATS_DROP_NOMEM_PKT,		"DropNoMemPkt",		"\t\t%llu dropped due to buflet alloc failure\n")       \
	X(NETIF_STATS_DROP_NOMEM_MBUF,		"DropNoMemMbuf",	"\t\t%llu dropped due to mbuf alloc failure\n") \
	X(NETIF_STATS_DROP_BADLEN,		"DropBadLen",		"\t\t%llu dropped due to bad packet length\n")  \
	X(NETIF_STATS_DROP_NA_INACTIVE,		"DropNaInactive",	"\t\t%llu dropped due to dst na inactive\n")    \
	X(NETIF_STATS_DROP_KRDROP_MODE,		"DropKrDropMode",	"\t\t%llu dropped due to dst kring in drop mode\n")     \
	X(NETIF_STATS_DROP_RXQ_OVFL,		"DropRxqOverflow",	"\t\t%llu dropped due to RX Queue overflow\n")     \
	X(NETIF_STATS_DROP_NO_RX_CB,		"DropNoRxCallback",	"\t\t%llu dropped due to missing RX callback\n") \
        \
	/* Channel event stats */  \
	X(NETIF_STATS_EV_SENT,			"EvSent",		"\t%llu channel event delivered\n")     \
	X(NETIF_STATS_EV_DROP,			"EvDrop",		"\t%llu channel event dropped\n")     \
	X(NETIF_STATS_EV_DROP_NOMEM_PKT,	"EvDropNoMemPkt",	"\t%llu channel event dropped due to packet alloc failure\n")     \
	X(NETIF_STATS_EV_DROP_NA_INACTIVE,	"EvDropNaInactive",	"\t%llu channel event dropped due to na inactive\n")     \
	X(NETIF_STATS_EV_DROP_NA_DEFUNCT,	"EvDropNaDefunct",	"\t%llu channel event dropped due to na defunct\n")     \
	X(NETIF_STATS_EV_DROP_KRDROP_MODE,	"EvDropKrDropMode",	"\t%llu channel event dropped due to dst kring in drop mode\n")     \
	X(NETIF_STATS_EV_DROP_KEVENT_INACTIVE,	"EvDropKevInactive",	"\t%llu channel event dropped due to kevent not registered on channel\n")     \
	X(NETIF_STATS_EV_DROP_KRSPACE,		"EvDropKrSpaceDrop",	"\t%llu channel event dropped due to lack of space in user channel ring\n") \
	X(NETIF_STATS_EV_DROP_DEMUX_ERR,	"EvDropDemuxErr",	"\t%llu channel event dropped due to demux error\n") \
	X(NETIF_STATS_EV_DROP_EV_VPNA_NOTSUP,	"EvDropVpnaEvNotSup",	"\t%llu channel event dropped due to vpna not having event ring\n") \
	X(NETIF_STATS_EV_DROP_NO_VPNA,		"EvDropNoVpna",		"\t%llu channel event dropped due to no pna ports\n") \
        \
	/* Interface advisory update stats */  \
	X(NETIF_STATS_IF_ADV_UPD_RECV,		"IfAdvUpdRecv",		"\t%llu interface advisory update received\n") \
	X(NETIF_STATS_IF_ADV_UPD_SENT,		"IfAdvUpdSent",		"\t%llu interface advisory update event sent\n") \
	X(NETIF_STATS_IF_ADV_UPD_DROP,		"IfAdvUpdDrop",		"\t%llu interface advisory update event dropped\n") \
        \
	/* Interface filter stats */ \
	X(NETIF_STATS_FILTER_DROP_NO_RX_CB,	"FilterDropNoRxCB",	"\t%llu dropped due to missing RX callback\n") \
	X(NETIF_STATS_FILTER_DROP_DISABLED,	"FilterDropDisabled",	"\t%llu dropped due to disabled filter\n") \
	X(NETIF_STATS_FILTER_DROP_REMOVED,	"FilterDropRemoved",	"\t%llu dropped due to removed filter\n") \
	X(NETIF_STATS_FILTER_DROP_PKTQ_FULL,	"FilterDropPktqFull",	"\t%llu dropped due to packet queue full\n") \
	X(NETIF_STATS_FILTER_DROP_MBQ_FULL,	"FilterDropMbqFull",	"\t%llu dropped due to mbuf queue full\n") \
	X(NETIF_STATS_FILTER_DROP_DISABLED_RING,"FilterDropDisabledRing","\t%llu dropped due to disabled ring\n") \
	X(NETIF_STATS_FILTER_DROP_NO_SPACE,	"FilterDropNoSpace",	"\t%llu dropped due to lack of space in RX ring\n") \
	X(NETIF_STATS_FILTER_DROP_INTERNALIZE,	"FilterDropInternalize","\t%llu dropped due to internalize failure\n") \
	X(NETIF_STATS_FILTER_DROP_PKT_ALLOC_FAIL,"FilterDropPktAllocFail","\t%llu dropped due to packet allocation failure\n") \
	X(NETIF_STATS_FILTER_DROP_DEFAULT,	"FilterDropDefault",	"\t%llu dropped due to default drop policy\n") \
	X(NETIF_STATS_FILTER_PKT_TRUNCATED,	"FilterPktTruncated",	"\t%llu inbound packets truncated\n") \
	X(NETIF_STATS_FILTER_TX_DELIVER,	"FilterTxDeliver",	"\t%llu outbound packets delivered to filter\n") \
	X(NETIF_STATS_FILTER_RX_DELIVER,	"FilterRxDeliver",	"\t%llu inbound packets delivered to filter\n") \
	X(NETIF_STATS_FILTER_TX_INJECT,		"FilterTxInject",	"\t%llu outbound packets injected by filter\n") \
	X(NETIF_STATS_FILTER_RX_INJECT,		"FilterRxInject",	"\t%llu inbound packets injected by filter\n") \
	X(NETIF_STATS_FILTER_TX_ENTER,		"FilterTxEnter",	"\t%llu outbound packets entered the filter chain\n") \
	X(NETIF_STATS_FILTER_RX_ENTER,		"FilterRxEnter",	"\t%llu inbound packets entered the filter chain\n") \
	X(NETIF_STATS_FILTER_TX_EXIT,		"FilterTxExit",		"\t%llu outbound packets exited the filter chain\n") \
	X(NETIF_STATS_FILTER_RX_EXIT,		"FilterRxExit",		"\t%llu inbound packets exited the filter chain\n") \
	X(NETIF_STATS_FILTER_SYNC_NO_PKTS,	"FilterSyncNoPkts",	"\t%llu filter syncs called with an empty ring\n") \
	X(NETIF_STATS_FILTER_ADD,		"FilterAdd",		"\t%llu filters added\n") \
	X(NETIF_STATS_FILTER_REMOVE,		"FilterRemove",		"\t%llu filters removed\n") \
	X(NETIF_STATS_FILTER_TX_FLUSH,		"FilterTxFlush",	"\t%llu TX packets flushed due to closing filters\n") \
	X(NETIF_STATS_FILTER_RX_NOT_FILTERABLE,	"FilterRxNotFilterable","\t%llu RX packets not filterable\n") \
	X(NETIF_STATS_FILTER_BAD_PKT_LEN,	"FilterBadPktLen",	"\t%llu invalid packet length\n") \
        \
	/* Custom ether and sidecar stats */ \
	X(NETIF_STATS_VP_DROP_USER_RING_DISABLED,"VPDropUserRingDisabled","\t%llu dropped due to disabled user ring\n") \
	X(NETIF_STATS_VP_DROP_DEV_RING_DISABLED,"VPDropDevRingDisabled","\t%llu dropped due to disabled device ring\n") \
	X(NETIF_STATS_VP_DROP_USER_RING_NO_SPACE,"VPDropUserRingNoSpace","\t%llu dropped due to lack of user ring space\n") \
	X(NETIF_STATS_VP_DROP_DEV_RING_NO_SPACE,"VPDropDevRingNoSpace",	"\t%llu dropped due to lack of device ring space\n") \
	X(NETIF_STATS_VP_DROP_RX_ALLOC_FAIL,	"VPDropRxAllocFail",	"\t%llu dropped due to RX allocation failure\n") \
	X(NETIF_STATS_VP_DROP_TX_ALLOC_FAIL,	"VPDropTxAllocFail",	"\t%llu dropped due to TX allocation failure\n") \
	X(NETIF_STATS_VP_DROP_PKT_TOO_BIG,	"VPDropPktTooBig",	"\t%llu dropped due to packet being too big\n") \
	X(NETIF_STATS_VP_DROP_INTERNALIZE_FAIL,	"VPDropInternalizeFail","\t%llu dropped due to internalize failure\n") \
	X(NETIF_STATS_VP_DROP_UNEXPECTED_ERR,	"VPDropUnexpectedErr",	"\t%llu dropped due to unexpected TX sync error\n") \
	X(NETIF_STATS_VP_BAD_MADDR_LEN,		"VPBadMaddrLen",	"\t%llu packets with invalid mac address length\n") \
	X(NETIF_STATS_VP_BAD_MADDR,		"VPBadMaddr",		"\t%llu packets with invalid mac address\n") \
	X(NETIF_STATS_VP_BAD_PKT_LEN,		"VPBadPktLen",		"\t%llu packets invalid packet length\n") \
	X(NETIF_STATS_VP_FLOW_INFO_ERR,		"VPFlowInfoErr",	"\t%llu packets cannot be classified due to flow info error\n") \
	X(NETIF_STATS_VP_FLOW_NOT_MATCH,	"VPFlowNotMatch",	"\t%llu packets not matching flow description\n") \
	X(NETIF_STATS_VP_KR_ENTER_FAIL,		"VPKrEnterFail",	"\t%llu failed attempts to acquire RX ring lock\n") \
	X(NETIF_STATS_VP_DEV_RING_DISABLED,	"VPDevRingDisabled",	"\t%llu failed attempts to get packets due to disabled dev ring\n") \
	X(NETIF_STATS_VP_SYNC_UNKNOWN_ERR,	"VPSyncUnknownErr",	"\t%llu unknown errors returned by RX sync\n") \
	X(NETIF_STATS_VP_SYNC_NO_PKTS,		"VPSyncNoPkts",		"\t%llu syncs called with an empty ring\n") \
	X(NETIF_STATS_VP_SPURIOUS_NOTIFY,	"VPSpuriousNotify",	"\t%llu spurious notifies delivered\n") \
	X(NETIF_STATS_VP_ENQUEUE_FAILED,	"VPEnqueueFailed",	"\t%llu packets failed to enqueue\n") \
	X(NETIF_STATS_VP_ENQUEUED,		"VPEnqueued",		"\t%llu packets enqueued\n") \
	X(NETIF_STATS_VP_LL_ENQUEUED,		"VPLLEnqueued",		"\t%llu low latency packets enqueued\n") \
	X(NETIF_STATS_VP_LL_SENT,		"VPLLSent",		"\t%llu low latency packets sent\n") \
	X(NETIF_STATS_VP_LL_DELIVERED,		"VPLLDelivered",	"\t%llu low latency packets delivered\n") \
	X(NETIF_STATS_VP_DELIVERED,		"VPDelivered",		"\t%llu packets delivered\n") \
	X(NETIF_STATS_VP_FLOW_FOUND,		"VPFlowFound",		"\t%llu packets found a matching flow\n") \
	X(NETIF_STATS_VP_FLOW_NOT_FOUND,	"VPFlowNotFound",	"\t%llu packets found no matching flow\n") \
	X(NETIF_STATS_VP_FLOW_DISABLED,		"VPFlowDisabled",	"\t%llu lookup failures due to disabled flow\n") \
	X(NETIF_STATS_VP_FLOW_EMPTY_TABLE,	"VPFlowEmptyTable",	"\t%llu lookup failures due to empty flow table\n") \
	X(NETIF_STATS_VP_FLOW_TABLE_INIT_FAIL,	"VPFlowTableInitFail",	"\t%llu failed attempts to initialize flow table\n") \
	X(NETIF_STATS_VP_FLOW_INSERT_FAIL,	"VPFlowInsertFail",	"\t%llu failed attempts to insert flow\n") \
	X(NETIF_STATS_VP_FLOW_ADD,		"VPFlowAdd",		"\t%llu flows added\n") \
	X(NETIF_STATS_VP_FLOW_REMOVE,		"VPFlowRemove",		"\t%llu flows removed\n") \
        \
	/* Netif agent stats */ \
	X(NETIF_STATS_AGENT_BAD_ETHERTYPE,	"AgentBadEthertype",	"\t%llu flow add failures due to invalid ethertype\n") \
	X(NETIF_STATS_AGENT_BAD_IPV6_ADDR,	"AgentBadIPv6Addr",	"\t%llu flow add failures due to invalid IPv6 address\n") \
	X(NETIF_STATS_AGENT_DUP_FLOW,		"AgentDupFlow",		"\t%llu duplicate flows added\n") \
        \
	/* Netif llink stats */ \
	X(NETIF_STATS_LLINK_ADD,		"LLinkAdd",		"\t%llu logical links added\n") \
	X(NETIF_STATS_LLINK_REMOVE,		"LLinkRemove",		"\t%llu logical links removed\n") \
	X(NETIF_STATS_LLINK_DEF_QSET_USED,	"LLinkDefQSetUsed",	"\t%llu uses of the default qset\n") \
	X(NETIF_STATS_LLINK_NONDEF_QSET_USED,	"LLinkNonDefQSetUsed",	"\t%llu uses of a non-default qset\n") \
	X(NETIF_STATS_LLINK_HINT_NOT_USEFUL,	"LLinkHintNotUseful",	"\t%llu hints specified but qset not found\n") \
	X(NETIF_STATS_LLINK_DUP_INT_ID_GENERATED, "LLinkDupIntIDGenerated", "\t%llu duplicate internal llink IDs generated\n") \
	X(NETIF_STATS_LLINK_DUP_ID_GIVEN,	"LLinkDupIDGiven",	"\t%llu duplicate llink IDs given by the provider\n") \
	X(NETIF_STATS_LLINK_QSET_INIT_FAIL,	"LLinkQSetInitFail",	"\t%llu queue set initialization failures\n") \
	X(NETIF_STATS_LLINK_RXQ_INIT_FAIL,	"LLinkRXQInitFail",	"\t%llu RX queue initialization failures\n") \
	X(NETIF_STATS_LLINK_TXQ_INIT_FAIL,	"LLinkTXQInitFail",	"\t%llu TX queue initialization failures\n") \
	X(NETIF_STATS_LLINK_NOT_FOUND_REMOVE,	"LLinkNotFoundRemove",	"\t%llu not found during remove\n") \
	X(NETIF_STATS_LLINK_TX_DROP_BAD_STATE,	"LLinkTxDroppedBadState", "\t%llu TX packets dropped due to bad llink state\n") \
	X(NETIF_STATS_LLINK_RX_DROP_BAD_STATE,	"LLinkRxDroppedBadState", "\t%llu RX packets dropped due to bad llink state\n") \
	X(NETIF_STATS_LLINK_AQM_QFULL,		"LLinkAQMQFull",	"\t%llu occurances of the queue full condition\n") \
	X(NETIF_STATS_LLINK_AQM_DROPPED,	"LLinkAQMDropped",	"\t%llu packets dropped due to AQM\n") \
	X(NETIF_STATS_LLINK_AQM_DEQ_BAD_STATE,	"LLinkAQMDeqBadState",	"\t%llu dequeues occurred while llink is in a bad state\n") \
	X(NETIF_STATS_LLINK_QSET_BAD_STATE,	"LLinkQSetAccessBadState", "\t%llu attempts to access a queue set while in bad llink state\n") \
	X(NETIF_STATS_LLINK_ADD_BAD_PARAMS,	"LLinkAddBadParams",	"\t%llu attempts to add an llink with bad parameters\n") \
        \
	X(__NETIF_STATS_MAX,			"",			"end of netif stats")

#define FSW_STATS_TABLE(X)                                      \
	/* Rx stats */  \
	X(FSW_STATS_RX_PACKETS,			"RxPackets",		"\t%llu total Rx packet\n")     \
	X(FSW_STATS_RX_DEMUX_ERR,		"RxDemuxErr",		"\t\t%llu demux error (passed to BSD)\n")       \
	X(FSW_STATS_RX_DEMUX_UNSPEC,		"RxDemuxUnspec",	"\t\t%llu demux AF unkown (passed to BSD)\n")       \
	X(FSW_STATS_RX_DEMUX_PROMISC,		"RxDemuxPromisc",	"\t\t%llu promiscuous packets (passed to BSD)\n")  \
	X(FSW_STATS_RX_FLOW_EXTRACT_ERR,	"RxFlowExtractErr",	"\t\t%llu flow extract error (passed to BSD)\n")       \
	X(FSW_STATS_RX_PKT_NOT_FINALIZED,	"RxPktNotFinalized",	"\t\t%llu packet not finalized\n")  \
	X(FSW_STATS_RX_FLOW_NOT_FOUND,		"RxFlowNotFound",	"\t\t%llu dropped, flow lookup failure\n")        \
	X(FSW_STATS_RX_FLOW_TRACK_ERR,		"RxFlowTrackErr",	"\t\t%llu dropped, flow tracker error\n") \
	X(FSW_STATS_RX_FLOW_NONVIABLE,		"RxFlowNonviable",	"\t\t%llu dropped, flow already nonviable\n") \
	X(FSW_STATS_RX_FLOW_TORNDOWN,		"RxFlowTornDown",	"\t\t%llu dropped, flow already torndown\n")      \
	X(FSW_STATS_RX_DST_RING_FULL,		"RxDstRingFull",	"\t\t%llu dropped, destination ring full\n") \
	X(FSW_STATS_RX_COPY_PKT2PKT,		"RxCopyPktToPkt",	"\t\t%llu copied pkt  -> pkt\n")        \
	X(FSW_STATS_RX_COPY_PKT2MBUF,		"RxCopyPktToMbuf",	"\t\t%llu copied pkt  -> mbuf\n")       \
	X(FSW_STATS_RX_COPY_MBUF2PKT,		"RxCopyMbufToPkt",	"\t\t%llu copied mbuf -> pkt\n")        \
	X(FSW_STATS_RX_COPY_SUM,		"RxCopySum",		"\t\t%llu copy+checksumed\n")   \
	X(FSW_STATS_RX_COPY_BAD_LEN,		"RxCopyBadLen",		"\t\t%llu dropped, bad packet length\n")  \
	X(FSW_STATS_RX_DROP_NOMEM_BUF,		"RxDropNoMemBuf",	"\t\t%llu dropped due to mbuf alloc failure\n") \
	/* Rx frag stats (fsw doesn't manage fragments on Tx) */        \
	X(FSW_STATS_RX_FRAG_V4,			"RxFragV4",		"\t\t%llu total received ipv4 fragments\n")     \
	X(FSW_STATS_RX_FRAG_V6,			"RxFragV6",		"\t\t%llu total received ipv6 fragments\n")     \
	X(FSW_STATS_RX_FRAG_REASSED,		"RxFragReassed",	"\t\t\t%llu frag successfully reassembled\n")   \
	X(FSW_STATS_RX_FRAG_DROP_NOSLOT,	"RxFragDropNoSlot",	"\t\t\t%llu dropped no slot in dring\n")        \
	X(FSW_STATS_RX_FRAG_BAD,		"RxFragBad",		"\t\t\t%llu dropped due to bad fragments\n")    \
	X(FSW_STATS_RX_FRAG_DROP_BAD_LEN,	"RxFragBadLen",		"\t\t\t%llu dropped due to bad fragment length\n")      \
	X(FSW_STATS_RX_FRAG_DROP_NOMEM,		"RxFragNoMem",		"\t\t\t%llu dropped due to no memory\n")        \
	X(FSW_STATS_RX_FRAG_DROP_TIMEOUT,	"RxFragTimeOut",	"\t\t\t%llu dropped due to time out\n") \
	X(FSW_STATS_RX_FRAG_DROP_FRAG_LIMIT,	"RxFragHitFragLimit",	"\t\t\t%llu dropped due to ipf max limit\n")    \
	X(FSW_STATS_RX_FRAG_DROP_QUEUE_LIMIT,	"RxFragHitQueueLimit",	"\t\t\t%llu dropped due to queue max limit\n")  \
	X(FSW_STATS_RX_FRAG_DROP_PER_QUEUE_LIMIT,"RxFragHitPerQueueLimit","\t\t\t%llu dropped due to ipf max per queue limit\n")        \
	/* Rx aggregation stats */                                      \
	X(FSW_STATS_RX_AGG_PKT2PKT,             "RxAggPktToPkt",        "\t\t%llu aggregated pkt  -> super pkt\n")             \
	X(FSW_STATS_RX_AGG_PKT2MBUF,            "RxAggPktToMbuf",       "\t\t%llu aggregated pkt  -> super mbuf\n")          \
	X(FSW_STATS_RX_AGG_MBUF2PKT,            "RxAggMbufToPkt",       "\t\t%llu aggregated mbuf  -> super pkt\n")          \
	X(FSW_STATS_RX_AGG_MBUF2MBUF,           "RxAggMbufToMbuf",      "\t\t%llu aggregated mbuf  -> super mbuf\n")       \
	X(FSW_STATS_RX_AGG_LIMIT,               "RxAggHitAggLimit",     "\t\t%llu reached aggregation limit\n") \
	X(FSW_STATS_RX_AGG_NO_HLEN_IP,          "RxAggNoHdrLenIP",      "\t\t%llu IP header length compare mismatch\n") \
	X(FSW_STATS_RX_AGG_NO_TTL_IP,           "RxAggNoTTLIP",         "\t\t%llu IP TTL compare mismatch\n") \
	X(FSW_STATS_RX_AGG_NO_TOS_IP,           "RxAggNoTOSIP",         "\t\t%llu IP TOS compare mismatch\n") \
	X(FSW_STATS_RX_AGG_NO_OFF_IP,           "RxAggNoOffsetIP",      "\t\t%llu IP offset compare mismatch\n") \
	X(FSW_STATS_RX_AGG_NO_OPT_IP,           "RxAggNoOptionIP",      "\t\t%llu IP option compare mismatch\n") \
	X(FSW_STATS_RX_AGG_MERGE_FASTPATH_IP,   "RxAggMergeFastpathIP", "\t\t%llu IP header merge via fastpath\n") \
	X(FSW_STATS_RX_AGG_MERGE_SLOWPATH_IP,   "RxAggMergeSlowpathIP", "\t\t%llu IP header merge via slowpath\n") \
	X(FSW_STATS_RX_AGG_MERGE_FASTPATH_TCP,  "RxAggMergeFastpathTCP", "\t\t%llu TCP header merge via fastpath\n") \
	X(FSW_STATS_RX_AGG_MERGE_SLOWPATH_TCP,  "RxAggMergeSlowpathTCP", "\t\t%llu TCP header merge via slowpath\n") \
	X(FSW_STATS_RX_AGG_OK_FASTPATH_TCP,     "RxAggFastpathTCP",     "\t\t%llu TCP aggregation via fastpath\n") \
	X(FSW_STATS_RX_AGG_OK_SLOWPATH_TCP,     "RxAggSlowpathTCP",     "\t\t%llu TCP aggregation via slowpath\n") \
	X(FSW_STATS_RX_AGG_NO_SHORT_TCP,        "RxAggNoShortTCP",      "\t\t%llu TCP packet too short for mask compare\n") \
	X(FSW_STATS_RX_AGG_NO_MASK_TCP,         "RxAggNoMaskTCP",       "\t\t%llu TCP mask compare mismatch\n") \
	X(FSW_STATS_RX_AGG_NO_HLEN_TCP,         "RxAggNoHdrLenTCP",     "\t\t%llu TCP header length compare mismatch\n") \
	X(FSW_STATS_RX_AGG_NO_ULEN_TCP,         "RxAggNoULenTCP",       "\t\t%llu TCP ulength compare mismatch\n") \
	X(FSW_STATS_RX_AGG_NO_SEQN_TCP,         "RxAggNoSeqNTCP",       "\t\t%llu TCP sequence number compare mismatch\n") \
	X(FSW_STATS_RX_AGG_NO_ACKWIN_TCP,       "RxAggNoAckWinTCP",     "\t\t%llu TCP ACK or window compare mismatch\n") \
	X(FSW_STATS_RX_AGG_NO_FLAGS_TCP,        "RxAggNoFlagsTCP",      "\t\t%llu TCP flags compare mismatch\n") \
	X(FSW_STATS_RX_AGG_NO_EXOPT_TCP,        "RxAggNoExtraOptionTCP",  "\t\t%llu TCP extra option compare mismatch\n") \
	X(FSW_STATS_RX_AGG_NO_OPTTS_TCP,        "RxAggNoOptionTStampTCP", "\t\t%llu TCP timestamp option compare mismatch\n") \
	X(FSW_STATS_RX_AGG_BAD_CSUM,            "RxAggIncorrectChecksum", "\t\t%llu Incorrect TCP/IP checksum\n") \
	X(FSW_STATS_RX_AGG_NO_SHORT_MBUF,       "RxAggNoShortMbuf",      "\t\t%llu mbuf too short for mask compare\n") \
        \
	/* Tx stats */  \
	X(FSW_STATS_TX_PACKETS,			"TXPackets",		"\t%llu total Tx packets\n")    \
	X(FSW_STATS_TX_DEMUX_ERR,		"TxDemuxErr",		"\t\t%llu dropped, demux error\n")        \
	X(FSW_STATS_TX_FLOW_EXTRACT_ERR,	"TxFlowExtractErr",	"\t\t%llu dropped, flow extract error\n") \
	X(FSW_STATS_TX_FRAG_BAD_ID,		"TxFragID",		"\t\t%llu dropped, invalid fragment ID\n")    \
	X(FSW_STATS_TX_FRAG_BAD_CONT,		"TxContFrag",		"\t\t%llu dropped, invalid continuation fragment\n")    \
	X(FSW_STATS_TX_FLOW_NOT_FOUND,		"TxFlowNotFound",	"\t\t%llu dropped, flow lookup failure\n")        \
	X(FSW_STATS_TX_FLOW_TRACK_ERR,		"TxFlowTrackErr",	"\t\t%llu dropped, flow tracker error\n") \
	X(FSW_STATS_TX_FLOW_BAD_ID,		"TxFLowBadID",		"\t\t%llu dropped, flow id invalid\n") \
	X(FSW_STATS_TX_FLOW_NONVIABLE,		"TxFlowNonviable",	"\t\t%llu dropped, flow already nonviable\n") \
	X(FSW_STATS_TX_FLOW_TORNDOWN,		"TxFlowTornDown",	"\t\t%llu dropped, flow already torndown\n") \
	X(FSW_STATS_TX_BAD_LISTENER,		"TxBadListener",	"\t\t%llu dropped, flow as listener only\n")      \
	X(FSW_STATS_TX_AQM_DROP,		"TxAQMDrop",		"\t\t%llu dropped, AQM enqueue failure\n")        \
	X(FSW_STATS_TX_RESOLV_PENDING,		"TxResolvePending",	"\t\t%llu pending resolve\n")   \
	X(FSW_STATS_TX_RESOLV_FAIL,		"TxResolveFail",	"\t\t%llu dropped due to resolution failure\n") \
	X(FSW_STATS_TX_RESOLV_STALE,		"TxResolveStale",	"\t\t%llu resolved using existing info\n")      \
	X(FSW_STATS_TX_COPY_PKT2PKT,		"TxCopyPktToPkt",	"\t\t%llu copied pkt  ->  pkt\n")       \
	X(FSW_STATS_TX_COPY_PKT2MBUF,		"TxCopyPktToMbuf",	"\t\t%llu copied pkt  ->  mbuf\n")      \
	X(FSW_STATS_TX_COPY_SUM,		"TxCopySum",		"\t\t%llu copy+checksumed\n")   \
	X(FSW_STATS_TX_COPY_BAD_LEN,		"TxCopyBadLen",		"\t\t%llu dropped, bad packet length\n")  \
        \
	/* Drop stats (generic bidirectional) */ \
	X(FSW_STATS_DROP,			"Drop",			"\t%llu total dropped\n")       \
	X(FSW_STATS_DROP_NOMEM_PKT,		"DropNoMemPkt",		"\t\t%llu dropped, packet alloc failure\n")       \
	X(FSW_STATS_DROP_NOMEM_MBUF,		"DropNoMemMbuf",	"\t\t%llu dropped, mbuf alloc failure\n") \
        \
	/* Misc. stats */ \
	X(FSW_STATS_FLOWS_ABORTED,		"FlowsAborted",		"\t%llu flow aborted\n")        \
	X(FSW_STATS_DST_NXPORT_INVALID,		"DestNexusPortInvalid", "\t%llu times dst nexus port invalid\n")        \
	X(FSW_STATS_DST_NXPORT_INACTIVE,	"DestNexusPortInactive","\t%llu times dst nexus port inactive\n")       \
	X(FSW_STATS_DST_NXPORT_DEFUNCT,		"DestNexusPortDefunct", "\t%llu times dst nexus port defunct\n")        \
	X(FSW_STATS_DST_RING_DROPMODE,		"DestRingDropMode",	"\t\t%llu dropped, dst kring in drop mode\n")     \
	X(FSW_STATS_CHAN_ERR_UPP_ALLOC,		"ChanErrUppAlloc",	"\t%llu user packet pool alloc failure\n")      \
	X(FSW_STATS_CHAN_DEFUNCT_SKIP,		"ChanDefunctSkipped",	"\t%llu defunct skipped due to outstanding packets\n")  \
        \
	X(_FSW_STATS_ERROR_INJECTIONS,		"_ErrorInjections",	"(\t%llu errors injected)\n")  \
	X(FSW_STATS_IF_ADV_UPD_SENT,		"IfAdvUpdSent",		"\t%llu interface advisory update event sent\n") \
        \
	X(FSW_STATS_IF_ADV_UPD_DROP,		"IfAdvUpdDrop",		"\t%llu interface advisory update event dropped\n") \
	X(__FSW_STATS_MAX,			"",			"end of flowswitch stats")

/* END CSTYLED */

/*
 * Common stats operation and macro
 */
#define EXPAND_TO_ENUMERATION(a, b, c) a,
#define EXPAND_TO_STRING(a, b, c) b,
#define EXPAND_TO_FORMAT(a, b, c) c,

#define DEFINE_STATS_STR_FUNC(type, table)                      \
__attribute__((always_inline))                                  \
static inline const char *                                      \
type##_str(enum _##type value)                                  \
{                                                               \
	static const char *table[] = {                          \
	    table(EXPAND_TO_STRING)                             \
	};                                                      \
	return (table[value]);                                  \
}

#define DEFINE_STATS_FMT_FUNC(type, table)                      \
__attribute__((always_inline))                                  \
static inline const char *                                      \
type##_fmt(enum _##type value)                                  \
{                                                               \
	static const char *table[] = {                          \
	    table(EXPAND_TO_FORMAT)                             \
	};                                                      \
	return (table[value]);                                  \
}

#define STATS_VAL(s_ptr, t) ((s_ptr)->_arr[(t)])
#define STATS_INC(s_ptr, t) ((s_ptr)->_arr[(t)]++)
#define STATS_DEC(s_ptr, t) ((s_ptr)->_arr[(t)]--)
#define STATS_ADD(s_ptr, t, v) ((s_ptr)->_arr[(t)] += (v))

static inline void __attribute__((always_inline))
__stats_fold(uint64_t *dst, uint64_t *src, size_t len)
{
	// TODO replace with vector instruction once veclib is ready for xnu
	size_t i;
	for (i = 0; i < len; i++) {
		dst[i] += src[i];
	}
}

#define DEFINE_STATS_FOLD_FUNC(type, len)               \
static inline void __attribute__((always_inline))       \
type##_fold(struct type *dst, struct type *src)         \
{                                                       \
	__stats_fold(dst->_arr, src->_arr, len);        \
}

static inline void __attribute__((always_inline))
__stats_reset(uint64_t *_arr, size_t len)
{
	// TODO replace with vector instruction once veclib is ready for xnu
	size_t i;
	for (i = 0; i < len; i++) {
		_arr[i] = 0;
	}
}

#define DEFINE_STATS_RESET_FUNC(type, len)              \
static inline void __attribute__((always_inline))       \
type##_reset(struct type *s)                            \
{                                                       \
	__stats_reset(s->_arr, len);                    \
}

#define STATS_ALIGN 16  /* align for vector instruction */

#define STATS_REGISTER(name, NAME)                      \
enum _##name { NAME##_TABLE(EXPAND_TO_ENUMERATION) };   \
struct name {                                           \
	uint64_t	_arr[__##NAME##_MAX];           \
} __attribute__((aligned(STATS_ALIGN)));                \
DEFINE_STATS_STR_FUNC(name, NAME##_TABLE)               \
DEFINE_STATS_FMT_FUNC(name, NAME##_TABLE)               \
DEFINE_STATS_FOLD_FUNC(name, __##NAME##_MAX)            \
DEFINE_STATS_RESET_FUNC(name, __##NAME##_MAX)

/* Stats registration stub */
STATS_REGISTER(ip_stats, IP_STATS);
STATS_REGISTER(ip6_stats, IP6_STATS);
STATS_REGISTER(tcp_stats, TCP_STATS);
STATS_REGISTER(udp_stats, UDP_STATS);
STATS_REGISTER(quic_stats, QUIC_STATS);

STATS_REGISTER(fsw_stats, FSW_STATS);
STATS_REGISTER(netif_stats, NETIF_STATS);

#undef  STATS_REGISTER
#undef  DEFINE_STATS_RESET_FUNC
#undef  DEFINE_STATS_FOLD_FUNC
#undef  DEFINE_STATS_RESET_FUNC
#undef  DEFINE_STATS_FOLD_FUNC
#undef  DEFINE_STATS_STR_FUNC
#undef  DEFINE_STATS_STR_FUNC
#undef  EXPAND_TO_STRING
#undef  EXPAND_TO_ENUMERATION

/*
 * Channel/Ring stats
 */
typedef struct {
	uint32_t        cres_pkt_alloc_failures;
	uint32_t        __cres_reserved[1];
} channel_ring_error_stats, *channel_ring_error_stats_t;

typedef struct {
	uint64_t        crsu_total_slots_transferred;
	uint64_t        crsu_total_bytes_transferred;
	uint64_t        crsu_number_of_syncs;
	uint32_t        crsu_min_slots_transferred;
	uint32_t        crsu_max_slots_transferred;
	uint32_t        crsu_slots_per_sync;
	uint32_t        crsu_slots_per_sync_ma;
	uint64_t        crsu_bytes_per_sync;
	uint64_t        crsu_bytes_per_sync_ma;
	uint32_t        __crsu_reserved[2];
} channel_ring_user_stats, *channel_ring_user_stats_t;

typedef struct {
	uint64_t        crs_total_slots_transferred;
	uint64_t        crs_total_bytes_transferred;
	uint64_t        crs_number_of_transfers;
	uint32_t        crs_min_slots_transferred;
	uint32_t        crs_max_slots_transferred;
	uint32_t        crs_slots_per_second;
	uint32_t        crs_slots_per_second_ma;
	uint64_t        crs_bytes_per_second;
	uint64_t        crs_bytes_per_second_ma;
	uint32_t        __crs_reserved[2];
} channel_ring_stats, *channel_ring_stats_t;

/*
 * Nexus provider information.
 * Provides information about the provider including any registered instances.
 * Used with "kern.skywalk.nexus_provider_list" sysctl.
 */
#define NEXUS_PROVIDER_LIST_SYSCTL      "kern.skywalk.nexus_provider_list"

typedef struct {
	uuid_t          npi_prov_uuid;          /* nexus provider UUID */
	struct nxprov_params npi_prov_params;   /* nexus provider parameters */
	uint32_t        npi_instance_uuids_count;
	uint32_t        __npi_align_reserved;
	uuid_t          npi_instance_uuids[0];  /* nexus instance UUIDs */
} nexus_provider_info, *nexus_provider_info_t;

#define NEXUS_PROVIDER_INFO_SIZE(a) \
	offsetof(nexus_provider_info, npi_instance_uuids[a])

/*
 * Nexus channel information.
 * Provides information about every channel in the system.
 * Used with "kern.skywalk.nexus_channel_list" sysctl.
 */
#define NEXUS_CHANNEL_LIST_SYSCTL       "kern.skywalk.nexus_channel_list"

/* Enable/Disable channel ring stats collection */
#define NEXUS_CHANNEL_RING_STAT_ENABLE_SYSCTL   "kern.skywalk.ring_stat_enable"

typedef struct {
	ring_id_t               ncre_ring_id;
	uint32_t                __ncre_align_reserved;
	channel_ring_stats              ncre_stats;
	channel_ring_user_stats         ncre_user_stats;
	channel_ring_error_stats        ncre_error_stats;
} nexus_channel_ring_entry, *nexus_channel_ring_entry_t;

typedef struct {
	uuid_t          nce_uuid;       /* channel uuid */
	uint32_t        nce_flags;      /* SCHF_* */
	pid_t           nce_pid;        /* channel owner pid */
	int             nce_fd;         /* channel pid */
	nexus_port_t    nce_port;       /* connected nexus port */
	uint32_t        nce_tx_rings;   /* num of tx rings */
	uint32_t        nce_rx_rings;   /* num of rx rings */
	uint32_t        __nce_align_reserved;
	nexus_channel_ring_entry nce_ring_entries[0]; /* tx followed by rx */
} nexus_channel_entry, *nexus_channel_entry_t;

#define SCHF_MONITOR_TX         0x00000001
#define SCHF_MONITOR_RX         0x00000002
#define SCHF_MONITOR_NO_COPY    0x00000004
#define SCHF_USER_PACKET_POOL   0x00000008
#define SCHF_DEFUNCT_OK         0x00000010
#define SCHF_EXCLUSIVE          0x00000020
#define SCHF_FILTER             0x00000040
#define SCHF_EVENT_RING         0x00000080
#define SCHF_IF_ADV             0x00000100
#define SCHF_DEFUNCT_SKIP       0x00000200
#define SCHF_LOW_LATENCY        0x00000400
#define SCHF_CLOSING            0x40000000
#define SCHF_DEFUNCT            0x80000000

#define SCHF_BITS                                                          \
	"\020\01MON_TX\02MON_RX\03NO_COPY\04USER_PACKET_POOL\05DEFUNCT_OK" \
	"\06EXCLUSIVE\07FILTER\010EVENT_RING\011IF_ADV\012DEFUNCT_SKIP"    \
	"\013LOW_LATENCY\037CLOSING\040DEFUNCT"

#define NEXUS_CHANNEL_ENTRY_SIZE(n_rings)               \
	offsetof(nexus_channel_entry, nce_ring_entries[n_rings])

typedef struct {
	uuid_t          nci_instance_uuid;      /* nexus instance UUID */
	uint32_t        nci_channel_entries_count;
	uint32_t        __nci_align_reserved;
	nexus_channel_entry     nci_channel_entries[0]; /* variable length */
} nexus_channel_info, *nexus_channel_info_t;

/*
 * Nexus statistics types.
 */
typedef enum {
	NEXUS_STATS_TYPE_INVALID = 0,   /* invalid type */
	NEXUS_STATS_TYPE_FSW,           /* flowswitch */
	NEXUS_STATS_TYPE_CHAN_ERRORS,   /* Channel error stats */
} nexus_stats_type_t;

/*
 * Flowswitch statistics (NEXUS_STATS_TYPE_FSW).
 */
struct __nx_stats_fsw {
	struct ip_stats                 nxs_ipstat;
	struct ip6_stats                nxs_ip6stat;
	struct tcp_stats                nxs_tcpstat;
	struct udp_stats                nxs_udpstat;
	struct quic_stats               nxs_quicstat;
};

/*
 * Channel error statistics
 */
struct __nx_stats_channel_errors {
	channel_ring_error_stats_t      nxs_cres;
};

/*
 * Nexus advisories.
 */
#define NX_INTF_ADV_SIZE (sizeof(uint64_t) + sizeof(struct ifnet_interface_advisory))

#if defined(LIBSYSCALL_INTERFACE) || defined(BSD_KERNEL_PRIVATE)
/*
 * Nexus advisory region types.
 */
typedef enum {
#if defined(BSD_KERNEL_PRIVATE)
	NEXUS_ADVISORY_TYPE_INVALID = 0,
#endif /* BSD_KERNEL_PRIVATE */
	NEXUS_ADVISORY_TYPE_FLOWSWITCH = 1,   /* struct sk_nexusadv */
	NEXUS_ADVISORY_TYPE_NETIF,   /* struct netif_nexus_advisory */
} nexus_advisory_type_t;

/*
 * The metadata object is placed at the begining of nexus advisory region.
 */
struct __kern_nexus_adv_metadata {
	uint16_t                 knam_version;
	uint16_t                 __reserved;
	nexus_advisory_type_t    knam_type;
};
#define NX_ADVISORY_MD_VERSION            1
#define NX_ADVISORY_MD_CURRENT_VERSION    NX_ADVISORY_MD_VERSION

struct __kern_netif_intf_advisory {
	uint32_t                           cksum;
	uint32_t                           _reserved;
	struct ifnet_interface_advisory    adv;
} __attribute__((aligned(sizeof(uint64_t))));

/*
 * Netif nexus advisory.
 * currently this structure is not exposed to the user, but in future we
 * can expose it via os_channel_get_advisory_region() API.
 */
struct netif_nexus_advisory {
	uint64_t nna_version;
	/*
	 * __nna_intf_adv has been defined as an opaque blob here as the
	 * atomicity of the data can be guaranteed only if accessed using
	 * the os_channel_get_interface_advisory() API.
	 */
	union {
#if defined(LIBSYSCALL_INTERFACE) || defined(BSD_KERNEL_PRIVATE)
		struct __kern_netif_intf_advisory __kern_intf_adv;
#endif /* LIBSYSCALL_INTERFACE || BSD_KERNEL_PRIVATE */
		uint8_t __nna_intf_adv[NX_INTF_ADV_SIZE];
	};
} __attribute__((aligned(sizeof(uint64_t))));

/* Netif nexus advisory version */
#define NX_NETIF_ADVISORY_VERSION            1
#define NX_NETIF_ADVISORY_CURRENT_VERSION    NX_NETIF_ADVISORY_VERSION
#endif /* LIBSYSCALL_INTERFACE || BSD_KERNEL_PRIVATE */

/*
 * Flowswitch nexus advisory.
 *
 * Note: add new field members at the bottom; if layout changes in the
 * middle, bump up NX_ADVISORY_VERSION and recompile userland clients
 * accessing this structure.
 *
 * Timestamps are stored in nanoseconds, based on microuptime().
 * Use mach_timebase_info() to acquire numerator and denominator,
 * and then compute current time using the following:
 *
 * uint64_t now = (mach_absolute_time() * tb_info.numer) / tb_info.denom;
 *
 * Comparisons can then be done against that current time.
 */
struct sk_nexusadv {
	uint64_t nxadv_ver;             /* see NX_ADVISORY_VERSION */
	uint64_t nxadv_fg_sendts;       /* foreground traffic timestamp */
	uint64_t nxadv_rt_sendts;       /* realtime traffic timestamp */
	union {
#if defined(LIBSYSCALL_INTERFACE) || defined(BSD_KERNEL_PRIVATE)
		struct __kern_netif_intf_advisory _nxadv_intf_adv;
#endif /* LIBSYSCALL_INTERFACE || BSD_KERNEL_PRIVATE */
		uint8_t _nxadv_reserved[NX_INTF_ADV_SIZE];
	};
} __attribute__((aligned(sizeof(uint64_t))));

/* Flowswitch nexus advisory version */
#define NX_ADVISORY_VERSION     1
#define NX_ADVISORY_VERSION_2   2
#define NX_ADVISORY_CURRENT_VERSION     NX_ADVISORY_VERSION_2
#define NX_FLOWSWITCH_ADVISORY_CURRENT_VERSION    NX_ADVISORY_CURRENT_VERSION

typedef enum {
	/*
	 * TCP states.
	 */
	SFT_STATE_CLOSED = 0,           /* closed */
	SFT_STATE_LISTEN,               /* listening for connection */
	SFT_STATE_SYN_SENT,             /* active, have sent SYN */
	SFT_STATE_SYN_RECEIVED,         /* have sent and rcvd SYN */
	SFT_STATE_ESTABLISHED,          /* established */
	SFT_STATE_CLOSE_WAIT,           /* rcvd FIN, waiting close */
	SFT_STATE_FIN_WAIT_1,           /* have sent FIN */
	SFT_STATE_CLOSING,              /* exchanged FINs, waiting FIN|ACK */
	SFT_STATE_LAST_ACK,             /* rcvd FIN, closed, waiting FIN|ACK */
	SFT_STATE_FIN_WAIT_2,           /* closed, FIN is ACK'd */
	SFT_STATE_TIME_WAIT,            /* quiet wait after close */

	/*
	 * UDP states.
	 */
	SFT_STATE_NO_TRAFFIC = 20,      /* no packet observed */
	SFT_STATE_SINGLE,               /* single packet */
	SFT_STATE_MULTIPLE,             /* multiple packets */

	SFT_STATE_MAX = 255
} sk_stats_flow_track_state_t;

struct sk_stats_flow_track {
	uint64_t        sft_bytes;      /* bytes */
	uint64_t        sft_packets;    /* packets */
	uint64_t        sft_spackets;   /* super packets */
	sk_stats_flow_track_state_t sft_state;  /* SFT_STATE_* */
	uint32_t        sft_rtt;        /* avg ack rtt at flowswith */
	uint32_t        sft_seq;        /* max sequence number sent */
	uint16_t        sft_max_win;    /* largest window (pre scaling) */
	uint8_t         sft_wscale;     /* window scaling factor */
};

#define FLOW_STATS_IN_ADD(fe, stat, cnt) { \
	volatile struct sk_stats_flow_track *fst; \
	fst = &(fe)->fe_stats->fs_rtrack; \
	fst->sft_##stat += (cnt); \
}

#define FLOW_STATS_OUT_ADD(fe, stat, cnt) { \
	volatile struct sk_stats_flow_track *fst; \
	fst = &(fe)->fe_stats->fs_ltrack; \
	fst->sft_##stat += (cnt); \
}

/*
 * Skywalk flow (in kernel), equivalent of "net.inet.*.pcblist_n".
 * Output: Array of struct sk_stats_flow (per flow).
 */
#define SK_STATS_FLOW   "kern.skywalk.stats.flow"
struct sk_stats_flow {
	uuid_t          sf_nx_uuid;             /* nexus instance uuid */
	char            sf_if_name[IFNAMSIZ];   /* interface name */
	uint32_t        sf_if_index;            /* interface index */
	uint32_t        sf_bucket_idx;          /* flow bucket index */

	pid_t           sf_pid;                 /* flow pid */
	pid_t           sf_epid;                /* flow effective pid */
	char            sf_proc_name[32];       /* flow proc name */
	char            sf_eproc_name[32];      /* flow effecitve proc name */

	uint32_t        sf_flags;               /* SFLOWF_* */
	nexus_port_t    sf_nx_port;             /* nexus port */

	uint8_t         sf_protocol;            /* effective protocol */
	packet_svc_class_t sf_svc_class;        /* service class */
	flowadv_idx_t   sf_adv_idx;             /* flow advistory table index */
	struct flow_key sf_key __attribute__((__aligned__(16)));

	volatile struct sk_stats_flow_track sf_ltrack; /* local states */
	volatile struct sk_stats_flow_track sf_rtrack; /* remote states */
#define sf_obytes       sf_ltrack.sft_bytes
#define sf_opackets     sf_ltrack.sft_packets
#define sf_ospackets    sf_ltrack.sft_spackets
#define sf_ibytes       sf_rtrack.sft_bytes
#define sf_ipackets     sf_rtrack.sft_packets
#define sf_ispackets    sf_rtrack.sft_spackets
#define sf_lrtt         sf_ltrack.sft_rtt
#define sf_rrtt         sf_rtrack.sft_rtt
#define sf_lseq         sf_ltrack.sft_seq
#define sf_rseq         sf_rtrack.sft_seq
#define sf_lmax_win     sf_ltrack.sft_max_win
#define sf_rmax_win     sf_rtrack.sft_max_win
#define sf_lwscale      sf_ltrack.sft_wscale
#define sf_rwscale      sf_rtrack.sft_wscale

	activity_bitmap_t sf_activity;          /* flow activity bitmap */
};

/* valid values for sf_flags */
#define SFLOWF_TRACK            0x00000010      /* flow is tracked */
#define SFLOWF_CONNECTED        0x00000020      /* connected mode */
#define SFLOWF_LISTENER         0x00000040      /* listener mode */
#define SFLOWF_QOS_MARKING      0x00000100      /* flow can have qos marking */
#define SFLOWF_BOUND_IP         0x00000200      /* src addr explicity bound */
#define SFLOWF_ONLINK           0x00000400      /* dst directly on the link */
#define SFLOWF_LOW_LATENCY      0x00000800      /* low latency flow */
#define SFLOWF_WAIT_CLOSE       0x00001000      /* defer free after close */
#define SFLOWF_CLOSE_NOTIFY     0x00002000      /* notify NECP upon tear down */
#define SFLOWF_ABORTED          0x01000000      /* has sent RST to peer */
#define SFLOWF_NONVIABLE        0x02000000      /* disabled; to be torn down */
#define SFLOWF_WITHDRAWN        0x04000000      /* flow has been withdrawn */
#define SFLOWF_TORN_DOWN        0x08000000      /* torn down, to be destroyed */
#define SFLOWF_DESTROYED        0x40000000      /* not in RB trees anymore */
#define SFLOWF_LINGERING        0x80000000      /* destroyed and lingering */

#define SFLOW_BUCKET_NONE       ((uint32_t)-1)

#if defined(BSD_KERNEL_PRIVATE)
#include <os/refcnt.h>

/*
 * flow_stats is the kernel stats object that serves as efficient conduit
 * between stats producer (the Skywalk flowswitch) and consumers
 * (e.g. necp_client/ntstat). It embeds the sk_stats_flow along with a
 * reference count. The flow_stats object would be freed when released and
 * refcnt reaches 0. There must be only one producer and/or multiple consumers.
 * There is no lock protecting the stats object as inconsistent intermediate
 * state data is tolerable for stats consumers and most fields, e.g. integer
 * counters, are updated atomically. Synchronization of producer/consumer
 * should be done via other means, e.g. necp/ntstat events, rather than on the
 * flow_stats itself.
 *
 * Fields in flow_stats.fs_stats are published in different phases:
 *     - Descriptor fields
 *         ID, names, address, etc. which are immutable during flow lifetime,
 *         which are initialized during creation time.
 *     - State fields
 *         Flow state, track state, etc. which are mutable. They are
 *         initialized during flow creation time, but lazily updated during
 *         runtime and upon synchronous retrieval.
 *     - Runtime fields
 *         Counters (packets in/out, bytes in/out, rtt, etc.), which are
 *         mutable and updated in real-time.
 *
 * Note the reduced alignment for sk_stats_flow and sk_stats_flow_track to
 * reduce the allocation size.
 */
struct flow_stats {
	struct sk_stats_flow    fs_stats;
#define fs_ltrack       fs_stats.sf_ltrack
#define fs_rtrack       fs_stats.sf_rtrack
#define fs_activity     fs_stats.sf_activity
#define fs_lrtt         fs_stats.sf_lrtt
#define fs_rrtt         fs_stats.sf_rrtt

	os_refcnt_t             fs_refcnt;
};

extern void flow_stats_free(struct flow_stats *fs);

__attribute__((always_inline))
static inline void
flow_stats_retain(struct flow_stats *fs)
{
	os_ref_retain(&fs->fs_refcnt);
}

__attribute__((always_inline))
static inline void
flow_stats_release(struct flow_stats *fs)
{
	if (__improbable(os_ref_release(&fs->fs_refcnt) == 0)) {
		flow_stats_free(fs);
	}
}

__attribute__((always_inline))
static inline os_ref_count_t
flow_stats_refcnt(struct flow_stats *fs)
{
	return os_ref_get_count(&fs->fs_refcnt);
}
#endif /* BSD_KERNEL_PRIVATE */

#define SK_STATS_FLOW_OWNER     "kern.skywalk.stats.flow_owner"
struct sk_stats_flow_owner {
	uuid_t          sfo_nx_uuid;            /* nexus instance uuid */
	char            sfo_if_name[IFNAMSIZ];  /* attached interface name */
	uint32_t        sfo_bucket_idx;         /* flow owner bucket index */

	char            sfo_name[32];           /* flow owner name */
	pid_t           sfo_pid;                /* flow owner pid */

	nexus_port_t    sfo_nx_port;            /* flow owner nexus port */
	boolean_t       sfo_nx_port_pid_bound;  /* flow owner port pid bound */
	boolean_t       sfo_nx_port_destroyed;  /* flow owner port destroyed */
} __attribute__((aligned(64)));

#define SK_STATS_FLOW_ROUTE     "kern.skywalk.stats.flow_route"
struct sk_stats_flow_route {
	uuid_t          sfr_nx_uuid;            /* nexus instance UUID */
	uuid_t          sfr_uuid;               /* flow route UUID */
	char            sfr_if_name[IFNAMSIZ];  /* interface name */

	uint32_t        sfr_bucket_idx;         /* flow route bucket index */
	uint32_t        sfr_id_bucket_idx;      /* flow route id bucket index */

	uint32_t        sfr_flags;              /* SFLOWRTF_* */
	uint32_t        sfr_usecnt;             /* flow route usecnt */
	int64_t         sfr_expire;             /* seconds left to expire */

	union sockaddr_in_4_6 sfr_laddr;        /* local address */
	union sockaddr_in_4_6 sfr_faddr;        /* foreign address */
	union sockaddr_in_4_6 sfr_gaddr;        /* gateway address */

	uint8_t         sfr_ether_dhost[ETHER_ADDR_LEN]
	__attribute__((aligned(64)));
};

/* valid values for sfr_flags */
#define SFLOWRTF_ATTACHED       0x00000001 /* attached to RB trees */
#define SFLOWRTF_ONLINK         0x00000010 /* dst directly on the link */
#define SFLOWRTF_GATEWAY        0x00000020 /* gw IP address is valid */
#define SFLOWRTF_RESOLVED       0x00000040 /* flow route is resolved */
#define SFLOWRTF_HAS_LLINFO     0x00000080 /* has dst link-layer address */
#define SFLOWRTF_DELETED        0x00000100 /* route has been deleted */
#define SFLOWRTF_DST_LL_MCAST   0x00000200 /* dst is link layer multicast */
#define SFLOWRTF_DST_LL_BCAST   0x00000400 /* dst is link layer broadcast */

/*
 * Skywalk netif stats
 * Output: Array of struct sk_stats_net_if entry (per netif nexus instance).
 */
#define SK_STATS_NET_IF         "kern.skywalk.stats.net_if"
struct sk_stats_net_if {
	uuid_t          sns_nx_uuid;            /* nexus netif instance uuid */
	char            sns_if_name[IFNAMSIZ];  /* attached interface name */

	struct netif_stats      sns_nifs;       /* netif stats */
} __attribute__((aligned(64)));

/*
 * Skywalk flowswitch stats
 * Output: Array of struct sk_stats_flow_switch entry (per fsw nexus instance).
 */
#define SK_STATS_FLOW_SWITCH    "kern.skywalk.stats.flow_switch"
struct sk_stats_flow_switch {
	uuid_t          sfs_nx_uuid;            /* nexus fsw instance uuid */
	char            sfs_if_name[IFNAMSIZ];  /* attached interface name */

	struct fsw_stats        sfs_fsws;       /* flowswitch stats */
} __attribute__((aligned(64)));

/*
 * Skywalkuserstack stats
 *
 * With Skywalk, traditional kernel space stacks like tcp/udp/ip/ip6 are now
 * moved to userspace process. Together with this, stack statistics are now kept
 * per process via shared memory described by skywalk channel, which is in a
 * memory mapped region from a kernel nexus to a userspace process.
 *
 * Output: Array of struct sk_stats_userstack (per process, per nexus
 *         instance) Entries with nts_owner_pid == 0 are reserved for nexus
 *         stats collected from closed nexus ports.
 */
#define SK_STATS_USERSTACK      "kern.skywalk.stats.userstack"
struct sk_stats_userstack {
	uuid_t                  sus_nx_uuid;    /* nexus instance uuid */
	char            sus_if_name[IFNAMSIZ];  /* attached interface name */
	pid_t                   sus_owner_pid;  /* owner process */

	struct ip_stats         sus_ip;         /* process ip stats */
	struct ip6_stats        sus_ip6;        /* process ip6 stats */
	struct tcp_stats        sus_tcp;        /* process tcp stats */
	struct udp_stats        sus_udp;        /* process udp stats */
	struct quic_stats       sus_quic;       /* process quic stats */
} __attribute__((aligned(64)));

/*
 * Skywalk flow advisory table dump
 */
struct sk_stats_flow_adv_ent {
	uuid_t          sfae_flow_id;           /* flow ID */
	uint32_t        sfae_flags;             /* flags, FLOWADVF_* */
};

#define SK_STATS_FLOW_ADV       "kern.skywalk.stats.flow_adv"
struct sk_stats_flow_adv {
	uuid_t          sfa_nx_uuid;            /* nexus instance uuid */
	char            sfa_if_name[IFNAMSIZ];  /* attached interface name */
	pid_t           sfa_owner_pid;          /* owner process */

	uint32_t        sfa_entries_count;      /* number of flow adv entries */
	struct sk_stats_flow_adv_ent    sfa_entries[0]; /* flow adv entries */
};

/*
 * struct netns_ctl_dump_header is used for sysctl 'kern.skywalk.stats.netns'
 * which returns a buffer containing the contents of every netns namespace.
 * The buffer is formatted as a series headers, each immediately followed by
 * some number of records:
 *    { struct netns_ctl_dump_header h,
 *      struct netns_ctl_dump_record r[h->ncdh_n_records] } * total_namespaces
 */
#define SK_STATS_NETNS          "kern.skywalk.stats.netns"
struct netns_ctl_dump_header {
	union {
		uint32_t        ncdh_addr[4];
		struct in_addr  ncdh_inaddr;
		struct in6_addr ncdh_in6addr;
	};
	uint8_t         ncdh_addr_len;
	uint8_t         ncdh_proto;
	uint32_t        ncdh_n_records;
	/*
	 * In a 'kern.skywalk.stats.netns' response, followed by
	 * {ncdh_n_records} struct netns_ctl_dump_records
	 */
} __attribute__((aligned(32)));

struct netns_ctl_dump_record {
	in_port_t ncdr_port;
	in_port_t ncdr_port_end;
	uint32_t  ncdr_skywalk_refs;
	uint32_t  ncdr_bsd_refs;
	uint32_t  ncdr_pf_refs;
	uint32_t  ncdr_listener_refs;
} __attribute__((aligned(32)));

#define NS_DUMP_SIZE(records)   (sizeof (struct netns_ctl_dump_header) + \
	records * sizeof (struct netns_ctl_dump_record))

#define SK_STATS_PROTONS        "kern.skywalk.stats.protons"
struct sk_stats_protons_token {
	uint8_t                 spt_protocol;
	uint32_t                spt_refcnt;
	pid_t                   spt_pid;
	pid_t                   spt_epid;
};

typedef enum {
	/*
	 * The following are user task mappable.
	 */
	SREG_GUARD_HEAD = 0,    /* leading guard page(s) */
	SREG_SCHEMA,            /* channel layout */
	SREG_RING,              /* rings */
	SREG_BUF,               /* rx/tx buffers */
	SREG_RXBUF,             /* rx only buffers */
	SREG_TXBUF,             /* tx only buffers */
	SREG_UMD,               /* userland metadata */
	SREG_TXAUSD,            /* tx/alloc user slot descriptors */
	SREG_RXFUSD,            /* rx/free user slot descriptors */
	SREG_UBFT,              /* userland buflet metadata */
	SREG_USTATS,            /* statistics */
	SREG_FLOWADV,           /* flow advisories */
	SREG_NEXUSADV,          /* nexus advisories */
	SREG_SYSCTLS,           /* sysctl */
	SREG_GUARD_TAIL,        /* trailing guard page(s) */

	/*
	 * The following are NOT user task mappable.
	 */
	SREG_KMD,               /* rx/tx kernel metadata */
	SREG_RXKMD,             /* rx only kernel metadata */
	SREG_TXKMD,             /* tx only kernel metadata */
	SREG_KBFT,              /* rx/tx kernel buflet metadata */
	SREG_RXKBFT,            /* rx only kernel buflet metadata */
	SREG_TXKBFT,            /* tx only kernel buflet metadata */
	SREG_TXAKSD,            /* tx/alloc kernel slot descriptors */
	SREG_RXFKSD,            /* rx/free kernel slot descriptors */
	SREG_KSTATS,            /* kernel statistics snapshot */
	SREG_INSTRINSIC,        /* intrinsic objects */

	SREG_MAX                /* max */
} sk_stats_region_id_t;

#define SK_STATS_REGION         "kern.skywalk.stats.region"
struct sk_stats_region {
	/*
	 * Region properties.
	 */
	char            sreg_name[64];          /* region name */
	uuid_t          sreg_uuid;              /* region uuid */
	sk_stats_region_id_t sreg_id;           /* region ID */
	uint32_t        sreg_mode;              /* region mode flags */

	/*
	 * Region parameters.
	 */
	uint64_t        sreg_r_seg_size;        /* requested seg size */
	uint64_t        sreg_c_seg_size;        /* configured seg size */
	uint64_t        sreg_seg_cnt;           /* number of segments */
	uint64_t        sreg_seg_objs;          /* # of objects per segment */
	uint64_t        sreg_r_obj_size;        /* requested obj size */
	uint64_t        sreg_r_obj_cnt;         /* requested obj count */
	uint64_t        sreg_c_obj_size;        /* configured obj size */
	uint64_t        sreg_c_obj_cnt;         /* configured obj count */
	uint64_t        sreg_align;             /* object alignment */
	uint64_t        sreg_max_frags;         /* max number of buflets */

	/*
	 * Region statistics.
	 */
	uint64_t        sreg_meminuse;          /* memory in use */
	uint64_t        sreg_w_meminuse;        /* wired memory in use */
	uint64_t        sreg_memtotal;          /* total memory in region */
	uint64_t        sreg_seginuse;          /* total unfreed segments */
	uint64_t        sreg_rescale;           /* # of hash table rescales */
	uint64_t        sreg_hash_size;         /* size of hash table */
	uint64_t        sreg_alloc;             /* number of allocations */
	uint64_t        sreg_free;              /* number of frees */
};

/* valid values for sreg_mode */
#define SREG_MODE_NOREDIRECT    0x1     /* unaffected by defunct */
#define SREG_MODE_MMAPOK        0x2     /* can be mapped to user task */
#define SREG_MODE_KREADONLY     0x4     /* kernel read-only */
#define SREG_MODE_UREADONLY     0x8     /* if user map, map it read-only */
#define SREG_MODE_PERSISTENT    0x10    /* memory stays non-volatile */
#define SREG_MODE_MONOLITHIC    0x20    /* monolithic region */
#define SREG_MODE_NOMAGAZINES   0x40    /* disable magazines layer */
#define SREG_MODE_NOCACHE       0x80    /* caching-inhibited */
#define SREG_MODE_SEGPHYSCONTIG 0x100   /* phys. contiguous segment */
#define SREG_MODE_SHAREOK       0x200   /* allow object sharing */
#define SREG_MODE_IODIR_IN      0x400   /* I/O direction In */
#define SREG_MODE_IODIR_OUT     0x800   /* I/O direction Out */
#define SREG_MODE_GUARD         0x1000  /* guard pages region */
#define SREG_MODE_PUREDATA      0x2000  /* purely data; no pointers */
#define SREG_MODE_PSEUDO        0x4000  /* external backing store */
#define SREG_MODE_SLAB          (1U << 30) /* backend for slab layer */
#define SREG_MODE_MIRRORED      (1U << 31) /* controlled by another region */

#define SREG_MODE_BITS                                                  \
	"\020\01NOREDIRECT\02MMAPOK\03KREADONLY\04UREADONLY"            \
	"\05PERSISTENT\06MONOLITHIC\07NOMAGAZINES\10NOCACHE"            \
	"\11SEGPHYSCONTIG\012SHAREOK\013IODIR_IN\014IODIR_OUT"          \
	"\015GUARD\016PUREDATA\017PSEUDO\037SLAB\040MIRRORED"

typedef enum {
	SAR_TYPE_NEXUS,
	SAR_TYPE_NECP,
	SAR_TYPE_SYSTEM,
} sk_stats_arena_type_t;

#define SK_STATS_ARENA          "kern.skywalk.stats.arena"
struct sk_stats_arena {
	char                    sar_name[64];
	sk_stats_arena_type_t   sar_type;
	uint64_t                sar_mapsize;
	uuid_t                  sar_regions_uuid[SREG_MAX];
#define SK_STATS_ARENA_MAPPED_PID_MAX   8
	pid_t                   sar_mapped_pids[SK_STATS_ARENA_MAPPED_PID_MAX];
};

#define SK_STATS_CACHE          "kern.skywalk.stats.cache"
struct sk_stats_cache {
	/*
	 * Cache parameters.
	 */
	char            sca_name[64];           /* cache name */
	uuid_t          sca_uuid;               /* cache uuid */
	uuid_t          sca_ruuid;              /* backing region uuid */
	uint32_t        sca_mode;               /* cache mode flags */
	uint64_t        sca_bufsize;            /* object size */
	uint64_t        sca_objsize;            /* actual obj size in slab */
	uint64_t        sca_chunksize;          /* bufsize + alignment */
	uint64_t        sca_slabsize;           /* size of a slab */
	uint64_t        sca_bufalign;           /* buffer alignment */
	uint64_t        sca_objalign;           /* object alignment */

	/*
	 * Per-CPU caches statistics.
	 */
	uint64_t        sca_cpu_mag_size;       /* current magazine size */
	uint64_t        sca_cpu_mag_resize;     /* # of magazine resizes */
	uint64_t        sca_cpu_mag_purge;      /* # of magazine purges */
	uint64_t        sca_cpu_mag_reap;       /* # of magazine reaps */
	uint64_t        sca_depot_full;         /* # of full magazines */
	uint64_t        sca_depot_empty;        /* # of empty magazines */
	uint64_t        sca_depot_ws_zero;      /* # of working set flushes */
	uint64_t        sca_depot_contention_factor; /* contention factor */

	/*
	 * Slab statistics.
	 */
	uint64_t        sca_sl_create;          /* slab creates */
	uint64_t        sca_sl_destroy;         /* slab destroys */
	uint64_t        sca_sl_alloc;           /* slab layer allocations */
	uint64_t        sca_sl_free;            /* slab layer frees */
	uint64_t        sca_sl_alloc_fail;      /* total failed allocations */
	uint64_t        sca_sl_partial;         /* # of partial slabs */
	uint64_t        sca_sl_empty;           /* # of empty slabs */
	uint64_t        sca_sl_bufinuse;        /* total unfreed buffers */
	uint64_t        sca_sl_rescale;         /* # of hash table rescales */
	uint64_t        sca_sl_hash_size;       /* size of hash table */
};

/* valid values for sca_mode */
#define SCA_MODE_NOMAGAZINES    0x00000001      /* disable magazines layer */
#define SCA_MODE_AUDIT          0x00000002      /* audit transactions */
#define SCA_MODE_NOREDIRECT     0x00000004      /* unaffected by defunct */
#define SCA_MODE_BATCH          0x00000008      /* supports batch alloc/free */
#define SCA_MODE_DYNAMIC        0x00000010      /* enable magazine resizing */
#define SCA_MODE_CLEARONFREE    0x00000020      /* zero-out upon slab free */
#define SCA_MODE_PSEUDO         0x00000040      /* external backing store */

#define SCA_MODE_BITS \
	"\020\01NOMAGAZINES\02AUDIT\03NOREDIRECT\04BATCH\05DYNAMIC"     \
	"\06CLEARONFREE\07PSEUDO"

#endif /* PRIVATE || BSD_KERNEL_PRIVATE */
#endif /* !_SKYWALK_OS_STATS_H_ */
